m-labs · Apr 5, 2016 · Apr 5, 2016 · Apr 5, 2016 · Apr 5, 2016 · Apr 5, 2016
Showing with 31 additions and 74 deletions.

+0 −2 .gitignore

+11 −0 RELEASE_NOTES.rst

+3 −51 artiq/master/worker_db.py

+9 −15 artiq/master/worker_impl.py

+8 −6 artiq/test/test_h5types.py
diff --git a/.gitignore b/.gitignore
@@ -17,7 +17,6 @@ __pycache__/
 /misoc_*/
 
 /artiq/test/results
-/artiq/test/h5types.h5
 /artiq/examples/master/results
 /artiq/examples/master/last_rid.pyon
 /artiq/examples/master/dataset_db.pyon
@@ -31,5 +30,4 @@ __pycache__/
 /last_rid.pyon
 /dataset_db.pyon
 /device_db.pyon
-/h5types.h5
 /test*.py
diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst
@@ -3,6 +3,17 @@
 Release notes
 =============
 
+unreleased [1.0rc3]
+-------------------
+
+* The HDF5 format has changed.
+
+  * The datasets are located in the HDF5 subgroup ``datasets``.
+  * Datasets are now stored without additional type conversions and annotations
+    from ARTIQ, trusting that h5py maps and converts types between HDF5 and
+    python/numpy "as expected".
+
+
 1.0rc2
 ------
 

diff --git a/artiq/master/worker_db.py b/artiq/master/worker_db.py
@@ -157,56 +157,6 @@ def get_hdf5_output(start_time, rid, name):
     return h5py.File(os.path.join(dirname, filename), "w")
 
 
-_type_to_hdf5 = {
-    int: h5py.h5t.STD_I64BE,
-    float: h5py.h5t.IEEE_F64BE,
-
-    np.int8: h5py.h5t.STD_I8BE,
-    np.int16: h5py.h5t.STD_I16BE,
-    np.int32: h5py.h5t.STD_I32BE,
-    np.int64: h5py.h5t.STD_I64BE,
-
-    np.uint8: h5py.h5t.STD_U8BE,
-    np.uint16: h5py.h5t.STD_U16BE,
-    np.uint32: h5py.h5t.STD_U32BE,
-    np.uint64: h5py.h5t.STD_U64BE,
-
-    np.float16: h5py.h5t.IEEE_F16BE,
-    np.float32: h5py.h5t.IEEE_F32BE,
-    np.float64: h5py.h5t.IEEE_F64BE
-}
-
-def result_dict_to_hdf5(f, rd):
-    for name, data in rd.items():
-        flag = None
-        # beware: isinstance(True/False, int) == True
-        if isinstance(data, bool):
-            data = np.int8(data)
-            flag = "py_bool"
-        elif isinstance(data, int):
-            data = np.int64(data)
-            flag = "py_int"
-
-        if isinstance(data, np.ndarray):
-            dataset = f.create_dataset(name, data=data)
-        else:
-            ty = type(data)
-            if ty is str:
-                ty_h5 = "S{}".format(len(data))
-                data = data.encode()
-            else:
-                try:
-                    ty_h5 = _type_to_hdf5[ty]
-                except KeyError:
-                    raise TypeError("Type {} is not supported for HDF5 output"
-                                    .format(ty)) from None
-            dataset = f.create_dataset(name, (), ty_h5)
-            dataset[()] = data
-
-        if flag is not None:
-            dataset.attrs[flag] = np.int8(1)
-
-
 class DatasetManager:
     def __init__(self, ddb):
         self.broadcast = Notifier(dict())
@@ -240,4 +190,6 @@ def get(self, key):
             return self.ddb.get(key)
 
     def write_hdf5(self, f):
-        result_dict_to_hdf5(f, self.local)
+        g = f.create_group("datasets")
+        for k, v in self.local.items():
+            g[k] = v
diff --git a/artiq/master/worker_impl.py b/artiq/master/worker_impl.py
@@ -8,7 +8,8 @@
 import artiq
 from artiq.protocols import pipe_ipc, pyon
 from artiq.tools import multiline_log_config, file_import
-from artiq.master.worker_db import DeviceManager, DatasetManager, get_hdf5_output
+from artiq.master.worker_db import (DeviceManager, DatasetManager,
+                                    get_hdf5_output)
 from artiq.language.environment import is_experiment
 from artiq.language.core import set_watchdog_factory, TerminationRequested
 from artiq.coredevice.core import CompileError, host_only, _render_diagnostic
@@ -17,6 +18,7 @@
 
 ipc = None
 
+
 def get_object():
     line = ipc.readline().decode()
     return pyon.decode(line)
@@ -92,7 +94,7 @@ def pause(self):
     delete = staticmethod(make_parent_action("scheduler_delete"))
     request_termination = staticmethod(
         make_parent_action("scheduler_request_termination"))
-    get_status  = staticmethod(make_parent_action("scheduler_get_status"))
+    get_status = staticmethod(make_parent_action("scheduler_get_status"))
 
     def set_run_info(self, rid, pipeline_name, expid, priority):
         self.rid = rid
@@ -145,12 +147,6 @@ def examine(device_mgr, dataset_mgr, file):
             register_experiment(class_name, name, arginfo)
 
 
-def string_to_hdf5(f, key, value):
-    dtype = "S{}".format(len(value))
-    dataset = f.create_dataset(key, (), dtype)
-    dataset[()] = value.encode()
-
-
 def setup_diagnostics(experiment_file, repository_path):
     def render_diagnostic(self, diagnostic):
         message = "While compiling {}\n".format(experiment_file) + \
@@ -173,7 +169,8 @@ def render_diagnostic(self, diagnostic):
     # putting inherently local objects (the diagnostic engine) into
     # global slots, and there isn't any point in making it prettier by
     # wrapping it in layers of indirection.
-    artiq.coredevice.core._DiagnosticEngine.render_diagnostic = render_diagnostic
+    artiq.coredevice.core._DiagnosticEngine.render_diagnostic = \
+        render_diagnostic
 
 
 def main():
@@ -226,14 +223,11 @@ def main():
                 exp_inst.analyze()
                 put_object({"action": "completed"})
             elif action == "write_results":
-                f = get_hdf5_output(start_time, rid, exp.__name__)
-                try:
+                with get_hdf5_output(start_time, rid, exp.__name__) as f:
                     dataset_mgr.write_hdf5(f)
-                    string_to_hdf5(f, "artiq_version", artiq_version)
+                    f["artiq_version"] = artiq_version
                     if "repo_rev" in expid:
-                        string_to_hdf5(f, "repo_rev", expid["repo_rev"])
-                finally:
-                    f.close()
+                        f["repo_rev"] = expid["repo_rev"]
                 put_object({"action": "completed"})
             elif action == "examine":
                 examine(ExamineDeviceMgr, ParentDatasetDB, obj["file"])

diff --git a/artiq/test/test_h5types.py b/artiq/test/test_h5types.py
@@ -3,8 +3,6 @@
 import h5py
 import numpy as np
 
-from artiq.master.worker_db import result_dict_to_hdf5
-
 
 class TypesCase(unittest.TestCase):
     def test_types(self):
@@ -16,10 +14,14 @@ def test_types(self):
         }
 
         for size in 8, 16, 32, 64:
-            d["i"+str(size)] = getattr(np, "int" + str(size))(42)
-            d["u"+str(size)] = getattr(np, "uint" + str(size))(42)
+            for typ in "int", "uint":
+                dt = getattr(np, typ + str(size))
+                d[typ+str(size)] = dt(42)
+                d["n"+typ+str(size)] = np.array(42, dt)
+                d["m"+typ+str(size)] = np.array([[[[42]]]], dt)
         for size in 16, 32, 64:
             d["f"+str(size)] = getattr(np, "float" + str(size))(42)
 
-        with h5py.File("h5types.h5", "w") as f:
-            result_dict_to_hdf5(f, d)
+        with h5py.File("h5types.h5", "w", "core", backing_store=False) as f:
+            for k, v in d.items():
+                f[k] = v