tf input csv

Author: SaerdnaPp

0001_merge_file_line_num.py

@@ -1,7 +1,11 @@
 # -*- coding:utf-8 -*-
 
-f = open("data/merge/allRawData.txt")
+f = open("data/merge/allRawData.txt", "r")
 line_num = 0
+line = f.readline()
+str = line.split(" ")
+print len(str)
+print str[-1]
 for line in f:
 	line_num += 1
 print "File has %i lines" % (line_num)

0501_tf_input_csv.py 0501_tf_input_csv_test.py

@@ -2,19 +2,21 @@
 
 """
 @author: Songgx
-@file: 0501_tf_input_csv.py
+@file: 0501_tf_input_csv_test.py
 @time: 11/28/16 4:20 PM
 """
 
 from __future__ import print_function
 import tensorflow as tf
 
+
 def file_len(fname):
     with open(fname) as f:
         for i, l in enumerate(f):
             pass
     return i + 1
 
+
 filename = "data/csv_test_data.csv"
 
 # setup text reader
@@ -24,25 +26,25 @@ def file_len(fname):
 _, csv_row = reader.read(filename_queue)
 
 # setup CSV decoding
-record_defaults = [[0],[0],[0],[0],[0]]
-col1,col2,col3,col4,col5 = tf.decode_csv(csv_row, record_defaults=record_defaults)
+record_defaults = [[0], [0], [0], [0], [0]]
+col1, col2, col3, col4, col5 = tf.decode_csv(csv_row, record_defaults=record_defaults, field_delim=" ")
 
 # turn features back into a tensor
-features = tf.pack([col1,col2,col3,col4])
+features = tf.pack([col1, col2, col3, col4])
 
 print("loading, " + str(file_length) + " line(s)\n")
 with tf.Session() as sess:
-  tf.initialize_all_variables().run()
+    tf.initialize_all_variables().run()
 
-  # start populating filename queue
-  coord = tf.train.Coordinator()
-  threads = tf.train.start_queue_runners(coord=coord)
+    # start populating filename queue
+    coord = tf.train.Coordinator()
+    threads = tf.train.start_queue_runners(coord=coord)
 
-  for i in range(file_length):
-    # retrieve a single instance
-    example, label = sess.run([features, col5])
-    print(example, label)
+    for i in range(file_length):
+        # retrieve a single instance
+        example, label = sess.run([features, col5])
+        print(example, label)
 
-  coord.request_stop()
-  coord.join(threads)
-  print("\ndone loading")
+    coord.request_stop()
+    coord.join(threads)
+    print("\ndone loading")

0502_tf_input_merge_file.py

@@ -0,0 +1,50 @@
+# -*- coding:utf-8 -*-
+
+"""
+@author: Songgx
+@file: 0502_tf_input_merge_file.py
+@time: 11/28/16 5:04 PM
+"""
+
+from __future__ import print_function
+import tensorflow as tf
+
+
+def file_len(fname):
+    with open(fname) as f:
+        for i, l in enumerate(f):
+            pass
+    return i + 1
+
+
+filename = "data/csv_test_data.csv"
+
+# setup text reader
+file_length = file_len(filename)
+filename_queue = tf.train.string_input_producer([filename])
+reader = tf.TextLineReader(skip_header_lines=1)
+_, csv_row = reader.read(filename_queue)
+
+# setup CSV decoding
+record_defaults = [[0], [0], [0], [0], [0]]
+col1, col2, col3, col4, col5 = tf.decode_csv(csv_row, record_defaults=record_defaults, field_delim=" ")
+
+# turn features back into a tensor
+features = tf.pack([col1, col2, col3, col4])
+
+print("loading, " + str(file_length) + " line(s)\n")
+with tf.Session() as sess:
+    tf.initialize_all_variables().run()
+
+    # start populating filename queue
+    coord = tf.train.Coordinator()
+    threads = tf.train.start_queue_runners(coord=coord)
+
+    for i in range(file_length):
+        # retrieve a single instance
+        example, label = sess.run([features, col5])
+        print(example, label)
+
+    coord.request_stop()
+    coord.join(threads)
+    print("\ndone loading")

data/add_class_in_each_row.py

@@ -0,0 +1,51 @@
+# -*- coding:utf-8 -*-
+
+"""
+@author: Songgx
+@file: add_class_in_each_row.py
+@time: 11/28/16 7:20 PM
+"""
+
+TOTAL_ROW_NUM = 1000
+
+# line 0-999
+# line 0-99    class 0
+# line 100-199 class 1
+# ...
+# line 900-999 class 9
+
+class_num = 0
+line_num = 0
+
+fr = open("merge/allRawData.txt", "r")
+fw = open("merge/raw_data.txt", "w")
+for line in fr:
+    # line += str(class_num) not worked
+    line = line.strip() + " " + str(class_num) + "\n" # cancel "\n" and "\r"
+    fw.write(line)
+    line_num += 1
+    if line_num % 100 == 0:
+        class_num += 1
+        print "%i / 1000 lines finished." % (line_num)
+fr.close()
+fw.close()
+
+print "Verify new file:"
+fr1 = open("merge/raw_data.txt", "r")
+for i in range(1000):
+    line_num1 = i + 1
+    l = fr1.readline()[-10:]
+    if (line_num1+1) % 100 == 0 or (line_num1-1) % 100 == 0 or line_num1 % 100 == 0:
+        print "line-" + str(line_num1) + ":" + l.strip()
+fr1.close()
+print "Finished."
+
+
+
+
+
+
+
+
+
+

data/csv_test_data.csv

@@ -1,4 +1,5 @@
-0,0,0,0,0
-0,15,0,0,0
-0,30,0,0,0
-0,45,0,0,0
+0 15 0 0 0
+0 30 0 0 0
+0 45 0 0 0
+0 78 0 0 0
+0 65 0 12 1