validating ingest - kamangir/bolt#689

abcli/fashion_mnist.sh

@@ -21,7 +21,7 @@ function fashion_mnist() {
     if [ "$task" == "ingest" ] ; then
         python3 -m fashion_mnist \
             ingest \
-            --destination $abcli_object_path \
+            --output_path $abcli_object_path \
             ${@:2}
         return
     fi

fashion_mnist/__init__.py

@@ -1,5 +1,5 @@
 name = "fashion_mnist"
 
-version = "1.1.26"
+version = "1.1.33"
 
 description = "fashion-mnist + hugging-face + awesome-bash-cli"

fashion_mnist/__main__.py

@@ -1,5 +1,6 @@
 import argparse
 from . import *
+from .ingest import *
 from abcli import logging
 import logging
 
@@ -9,13 +10,18 @@ parser = argparse.ArgumentParser(name, description=f"{name}-{version}")
 parser.add_argument(
     "task",
     type=str,
-    help="TBD",
+    help="ingest",
+)
+parser.add_argument(
+    "--output_path",
+    type=str,
+    default="",
 )
 args = parser.parse_args()
 
 success = False
-if args.task == "TBD":
-    success = True
+if args.task == "ingest":
+    success = ingest(args.output_path)
 else:
     logger.error(f"-{name}: {args.task}: command not found.")
 

fashion_mnist/ingest.py

@@ -0,0 +1,57 @@
+from abcli import file
+from abcli import string
+from abcli.logging import crash_report
+import os.path
+from abcli import logging
+import tensorflow as tf
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def ingest(output_path):
+    import tensorflow as tf
+
+    try:
+        fashion_mnist = tf.keras.datasets.fashion_mnist
+
+        (train_images, train_labels), (
+            test_images,
+            test_labels,
+        ) = fashion_mnist.load_data()
+    except:
+        crash_report("-fashion_mnist: ingest.")
+        return False
+
+    logger.info("ingesting fashion_mnist")
+
+    success = True
+    for name, thing in zip(
+        "train_images,train_labels,test_images,test_labels".split(","),
+        [train_images, train_labels, test_images, test_labels],
+    ):
+        if file.save(os.path.join(output_path, name), thing):
+            logger.info(f"ingested {name}: {string.pretty_size_of_matrix(thing)}")
+        else:
+            success = False
+
+    class_names = [
+        "T-shirt/top",
+        "Trouser",
+        "Pullover",
+        "Dress",
+        "Coat",
+        "Sandal",
+        "Shirt",
+        "Sneaker",
+        "Bag",
+        "Ankle boot",
+    ]
+    if file.save_json(os.path.join(output_path, "class_names"), class_names):
+        logger.info(
+            f"ingested {len(class_names)} class name(s): {', '.join(class_names)}"
+        )
+    else:
+        success = False
+
+    return success