Swift: Pick an array elements randomly

I have an array of Int and I would like to pick a random Int from my array, with a specific conditions to fill a 2D grid. I tried the following code it run ok for many times but in some cases it's running with an infinite while loop :(

    for row in 0..<NumRows {
        for column in 0..<NumColumns {

                var gemType:GemType

                if gemArray.count == 1 {

                   gemType = GemType(rawValue: gemArray[0])!

                } else {

                    do {

                        index = Int(arc4random_uniform(UInt32(gemArray.count)))

                        gemType = GemType(rawValue: gemArray[index])!

                    } while (column >= 1 &&
                        gems[column - 1, row]?.gemType == gemType)
                        || (row >= 1 &&
                            gems[column, row - 1]?.gemType == gemType)

                    if gemArray.count > 0 {

                        gemArray.removeAtIndex(index)
                    }
                }

                let gem = Gem(column: column, row: row, gemType: gemType)
                gems[column, row] = gem

                // 4
                set.insert(gem)
            }
        }
    }
    return set

Any help?

Thanks!