I am trying to write a python class to read a subset of large number of images randomly.
Let's say we have about 1,250 images and each image is associated with another 12 sub-images. Each sub-image is associated with 2,000 indices. So there will be a total of 1250 x 12 x 2000 = 30,000,000 indices. Now given a list of indices (say 96 indices), we need to read from all the sub-images.
def read_image(filename):
return numpy.asarray(Image.open(filename))
def read_image_with_index(index, filename):
return index, read_image(filename)
def get_image_index(index):
return int(index/2000)
def read_in_seq(image_filenames, indices):
image_list = []
for index in indices:
image_index = get_image_index(index)
for suffix_index in range(12):
image_list.append( read_image(image_filenames[image_index+suffix_index]) )
return image_list
def read_in_parallel(image_filenames, indices):
image_groups = []
count = 0
for index in indices:
image_index = get_image_index(index)
for suffix_index in range(12):
image_groups.append( (count, image_filenames[image_index+suffix_index]) )
count = count + 1
image_list = [0] * len(image_groups)
with concurrent.futures.ThreadPoolExecutor(len(image_groups)) as executor:
images = {executor.submit(read_image_with_index, group[0], group[1]): group for group in image_groups}
for future in concurrent.futures.as_completed(images):
result = future.result()
image_list[result[0]] = result[1]
return image_list
Approach 1 (Normal read)
indices = numpy.asarray([30759704, 12267302, 27457073, 6384606, 1783414, 22502092, 14603110, 8821073])
# -- measure time start here
read_in_seq(input_image_list, indices) # total 8x12 sub-images
# -- measure time end here
Approach 2 (Random read)
batch_size = len(indices)
indices = numpy.arange(0 , 32762808)
numpy.random.shuffle(indices)
indices = indices[:batch_size]
# -- measure time start here
read_in_seq(input_image_list, indices) # total 8x12 sub-images
# -- measure time end here
Approach 3 (Multi-thread + Normal read)
indices = numpy.asarray([30759704, 12267302, 27457073, 6384606, 1783414, 22502092, 14603110, 8821073])
# -- measure time start here
read_in_parallel(input_image_list, indices) # total 8x12 sub-images
# -- measure time end here
Approach 4 (Multi-thread + Random read)
batch_size = len(indices)
indices = numpy.arange(0 , 32762808)
numpy.random.shuffle(indices)
indices = indices[:batch_size]
# -- measure time start here
read_in_parallel(input_image_list, indices) # total 8x12 sub-images
# -- measure time end here
So here is the time required for each approach:
approach 1: ~0.5 seconds
approach 2: ~2 seconds
approach 3: ~0.3 seconds
approach 4: ~5-7 seconds
So my questions are,
(1) Why the random read approach (without counting the time for numpy.random.shuffle) takes much longer time than normal read approach with hard-coded indices?
(2) Why the multi-thread approach takes much longer time than the single thread one?
Aucun commentaire:
Enregistrer un commentaire