Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: m-labs/artiq
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 4f589a72777c
Choose a base ref
...
head repository: m-labs/artiq
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: d6339e49ca86
Choose a head ref
  • 9 commits
  • 5 files changed
  • 1 contributor

Commits on Apr 5, 2016

  1. Copy the full SHA
    e1f3968 View commit details
  2. RELEASE_NOTES: style

    jordens committed Apr 5, 2016
    Copy the full SHA
    138271e View commit details
  3. Copy the full SHA
    a52b8fa View commit details
  4. Copy the full SHA
    4ba07e0 View commit details
  5. Copy the full SHA
    690eb8c View commit details
  6. Copy the full SHA
    0b32d99 View commit details
  7. RELEASE_NOTES: HDF5 encoding

    jordens committed Apr 5, 2016
    Copy the full SHA
    7dab043 View commit details
  8. worker_impl: style

    jordens committed Apr 5, 2016
    Copy the full SHA
    eea7cdc View commit details
  9. RELEASE_NOTES: spelling

    jordens committed Apr 5, 2016
    Copy the full SHA
    d6339e4 View commit details
Showing with 31 additions and 74 deletions.
  1. +0 −2 .gitignore
  2. +11 −0 RELEASE_NOTES.rst
  3. +3 −51 artiq/master/worker_db.py
  4. +9 −15 artiq/master/worker_impl.py
  5. +8 −6 artiq/test/test_h5types.py
2 changes: 0 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -17,7 +17,6 @@ __pycache__/
/misoc_*/

/artiq/test/results
/artiq/test/h5types.h5
/artiq/examples/master/results
/artiq/examples/master/last_rid.pyon
/artiq/examples/master/dataset_db.pyon
@@ -31,5 +30,4 @@ __pycache__/
/last_rid.pyon
/dataset_db.pyon
/device_db.pyon
/h5types.h5
/test*.py
11 changes: 11 additions & 0 deletions RELEASE_NOTES.rst
Original file line number Diff line number Diff line change
@@ -3,6 +3,17 @@
Release notes
=============

unreleased [1.0rc3]
-------------------

* The HDF5 format has changed.

* The datasets are located in the HDF5 subgroup ``datasets``.
* Datasets are now stored without additional type conversions and annotations
from ARTIQ, trusting that h5py maps and converts types between HDF5 and
python/numpy "as expected".


1.0rc2
------

54 changes: 3 additions & 51 deletions artiq/master/worker_db.py
Original file line number Diff line number Diff line change
@@ -157,56 +157,6 @@ def get_hdf5_output(start_time, rid, name):
return h5py.File(os.path.join(dirname, filename), "w")


_type_to_hdf5 = {
int: h5py.h5t.STD_I64BE,
float: h5py.h5t.IEEE_F64BE,

np.int8: h5py.h5t.STD_I8BE,
np.int16: h5py.h5t.STD_I16BE,
np.int32: h5py.h5t.STD_I32BE,
np.int64: h5py.h5t.STD_I64BE,

np.uint8: h5py.h5t.STD_U8BE,
np.uint16: h5py.h5t.STD_U16BE,
np.uint32: h5py.h5t.STD_U32BE,
np.uint64: h5py.h5t.STD_U64BE,

np.float16: h5py.h5t.IEEE_F16BE,
np.float32: h5py.h5t.IEEE_F32BE,
np.float64: h5py.h5t.IEEE_F64BE
}

def result_dict_to_hdf5(f, rd):
for name, data in rd.items():
flag = None
# beware: isinstance(True/False, int) == True
if isinstance(data, bool):
data = np.int8(data)
flag = "py_bool"
elif isinstance(data, int):
data = np.int64(data)
flag = "py_int"

if isinstance(data, np.ndarray):
dataset = f.create_dataset(name, data=data)
else:
ty = type(data)
if ty is str:
ty_h5 = "S{}".format(len(data))
data = data.encode()
else:
try:
ty_h5 = _type_to_hdf5[ty]
except KeyError:
raise TypeError("Type {} is not supported for HDF5 output"
.format(ty)) from None
dataset = f.create_dataset(name, (), ty_h5)
dataset[()] = data

if flag is not None:
dataset.attrs[flag] = np.int8(1)


class DatasetManager:
def __init__(self, ddb):
self.broadcast = Notifier(dict())
@@ -240,4 +190,6 @@ def get(self, key):
return self.ddb.get(key)

def write_hdf5(self, f):
result_dict_to_hdf5(f, self.local)
g = f.create_group("datasets")
for k, v in self.local.items():
g[k] = v
24 changes: 9 additions & 15 deletions artiq/master/worker_impl.py
Original file line number Diff line number Diff line change
@@ -8,7 +8,8 @@
import artiq
from artiq.protocols import pipe_ipc, pyon
from artiq.tools import multiline_log_config, file_import
from artiq.master.worker_db import DeviceManager, DatasetManager, get_hdf5_output
from artiq.master.worker_db import (DeviceManager, DatasetManager,
get_hdf5_output)
from artiq.language.environment import is_experiment
from artiq.language.core import set_watchdog_factory, TerminationRequested
from artiq.coredevice.core import CompileError, host_only, _render_diagnostic
@@ -17,6 +18,7 @@

ipc = None


def get_object():
line = ipc.readline().decode()
return pyon.decode(line)
@@ -92,7 +94,7 @@ def pause(self):
delete = staticmethod(make_parent_action("scheduler_delete"))
request_termination = staticmethod(
make_parent_action("scheduler_request_termination"))
get_status = staticmethod(make_parent_action("scheduler_get_status"))
get_status = staticmethod(make_parent_action("scheduler_get_status"))

def set_run_info(self, rid, pipeline_name, expid, priority):
self.rid = rid
@@ -145,12 +147,6 @@ def examine(device_mgr, dataset_mgr, file):
register_experiment(class_name, name, arginfo)


def string_to_hdf5(f, key, value):
dtype = "S{}".format(len(value))
dataset = f.create_dataset(key, (), dtype)
dataset[()] = value.encode()


def setup_diagnostics(experiment_file, repository_path):
def render_diagnostic(self, diagnostic):
message = "While compiling {}\n".format(experiment_file) + \
@@ -173,7 +169,8 @@ def render_diagnostic(self, diagnostic):
# putting inherently local objects (the diagnostic engine) into
# global slots, and there isn't any point in making it prettier by
# wrapping it in layers of indirection.
artiq.coredevice.core._DiagnosticEngine.render_diagnostic = render_diagnostic
artiq.coredevice.core._DiagnosticEngine.render_diagnostic = \
render_diagnostic


def main():
@@ -226,14 +223,11 @@ def main():
exp_inst.analyze()
put_object({"action": "completed"})
elif action == "write_results":
f = get_hdf5_output(start_time, rid, exp.__name__)
try:
with get_hdf5_output(start_time, rid, exp.__name__) as f:
dataset_mgr.write_hdf5(f)
string_to_hdf5(f, "artiq_version", artiq_version)
f["artiq_version"] = artiq_version
if "repo_rev" in expid:
string_to_hdf5(f, "repo_rev", expid["repo_rev"])
finally:
f.close()
f["repo_rev"] = expid["repo_rev"]
put_object({"action": "completed"})
elif action == "examine":
examine(ExamineDeviceMgr, ParentDatasetDB, obj["file"])
14 changes: 8 additions & 6 deletions artiq/test/test_h5types.py
Original file line number Diff line number Diff line change
@@ -3,8 +3,6 @@
import h5py
import numpy as np

from artiq.master.worker_db import result_dict_to_hdf5


class TypesCase(unittest.TestCase):
def test_types(self):
@@ -16,10 +14,14 @@ def test_types(self):
}

for size in 8, 16, 32, 64:
d["i"+str(size)] = getattr(np, "int" + str(size))(42)
d["u"+str(size)] = getattr(np, "uint" + str(size))(42)
for typ in "int", "uint":
dt = getattr(np, typ + str(size))
d[typ+str(size)] = dt(42)
d["n"+typ+str(size)] = np.array(42, dt)
d["m"+typ+str(size)] = np.array([[[[42]]]], dt)
for size in 16, 32, 64:
d["f"+str(size)] = getattr(np, "float" + str(size))(42)

with h5py.File("h5types.h5", "w") as f:
result_dict_to_hdf5(f, d)
with h5py.File("h5types.h5", "w", "core", backing_store=False) as f:
for k, v in d.items():
f[k] = v