tensorflow数据输入（文件输入）

1 tf_record_pattern = os.path.join(FLAGS.data_dir,'%s-*' %self.subset)  # subset in ['train','validation']
2 data_files = tf.gfile.Glob(tf_record_pattern)
或者
1 data_files = tf.train.match_filenames_once(tf.record_pattern)

 1 filename_queue = tf.train.string_input_producer(["file0.csv", "file1.csv"])
 2 
 3 reader = tf.TextLineReader()
 4 key, value = reader.read(filename_queue)
 5 
 6 # Default values, in case of empty columns. Also specifies the type of the
 7 # decoded result.
 8 record_defaults = [[1], [1], [1], [1], [1]]
 9 col1, col2, col3, col4, col5 = tf.decode_csv(
10     value, record_defaults=record_defaults)
11 features = tf.concat(0, [col1, col2, col3, col4])
12 
13 with tf.Session() as sess:
14   # Start populating the filename queue.
15   coord = tf.train.Coordinator()
16   threads = tf.train.start_queue_runners(coord=coord)
17 
18   for i in range(1200):
19     # Retrieve a single instance:
20     example, label = sess.run([features, col5])
21 
22   coord.request_stop()
23   coord.join(threads)

 1 def read_my_file_format(filename_queue):
 2   reader = tf.SomeReader()
 3   key, record_string = reader.read(filename_queue)
 4   example, label = tf.some_decoder(record_string)
 5   processed_example = some_processing(example)
 6   return processed_example, label
 7 
 8 def input_pipeline(filenames, batch_size, num_epochs=None):
 9   filename_queue = tf.train.string_input_producer(
10       filenames, num_epochs=num_epochs, shuffle=True)
11   example, label = read_my_file_format(filename_queue)　　#这里是单个reader在读文件，batch是当个reader读单个文件shuffle成batch
12   # min_after_dequeue defines how big a buffer we will randomly sample
13   #   from -- bigger means better shuffling but slower start up and more
14   #   memory used.
15   # capacity must be larger than min_after_dequeue and the amount larger
16   #   determines the maximum we will prefetch.  Recommendation:
17   #   min_after_dequeue + (num_threads + a small safety margin) * batch_size
18   min_after_dequeue = 10000
19   capacity = min_after_dequeue + 3 * batch_size
20   example_batch, label_batch = tf.train.shuffle_batch(
21       [example, label], batch_size=batch_size, capacity=capacity,
22       min_after_dequeue=min_after_dequeue)
23   return example_batch, label_batch

 1 def read_my_file_format(filename_queue):
 2   # Same as above
 3 
 4 def input_pipeline(filenames, batch_size, read_threads, num_epochs=None):
 5   filename_queue = tf.train.string_input_producer(
 6       filenames, num_epochs=num_epochs, shuffle=True) 
 7   example_list = [read_my_file_format(filename_queue)
 8                   for _ in range(read_threads)]           #这里是多个reader，batch_join将是多个reader读多个文件然后构成batch
 9   min_after_dequeue = 10000
10   capacity = min_after_dequeue + 3 * batch_size
11   example_batch, label_batch = tf.train.shuffle_batch_join(
12       example_list, batch_size=batch_size, capacity=capacity,
13       min_after_dequeue=min_after_dequeue)
14   return example_batch, label_batch