Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: m-labs/artiq
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 846701316073
Choose a base ref
...
head repository: m-labs/artiq
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: 5e14afde3ec0
Choose a head ref
  • 3 commits
  • 8 files changed
  • 1 contributor

Commits on Dec 6, 2015

  1. Copy the full SHA
    ea6dcb2 View commit details
  2. Copy the full SHA
    f99c53d View commit details
  3. Copy the full SHA
    5e14afd View commit details
4 changes: 2 additions & 2 deletions artiq/frontend/artiq_client.py
Original file line number Diff line number Diff line change
@@ -151,7 +151,7 @@ def _action_scan_devices(remote, args):


def _action_scan_repository(remote, args):
remote.scan_async(args.revision)
remote.scan_repository(args.revision)


def _show_schedule(schedule):
@@ -285,7 +285,7 @@ def main():
"set_dataset": "master_dataset_db",
"del_dataset": "master_dataset_db",
"scan_devices": "master_device_db",
"scan_repository": "master_repository"
"scan_repository": "master_experiment_db"
}[action]
remote = Client(args.server, port, target_name)
try:
6 changes: 3 additions & 3 deletions artiq/frontend/artiq_gui.py
Original file line number Diff line number Diff line change
@@ -66,7 +66,7 @@ def main():

# create connections to master
rpc_clients = dict()
for target in "schedule", "repository", "dataset_db":
for target in "schedule", "experiment_db", "dataset_db":
client = AsyncioClient()
loop.run_until_complete(client.connect_rpc(
args.server, args.port_control, "master_" + target))
@@ -99,14 +99,14 @@ def main():
sub_clients["explist"],
sub_clients["schedule"],
rpc_clients["schedule"],
rpc_clients["repository"])
rpc_clients["experiment_db"])
smgr.register(expmgr)
d_shortcuts = shortcuts.ShortcutsDock(win, expmgr)
smgr.register(d_shortcuts)
d_explorer = explorer.ExplorerDock(status_bar, expmgr, d_shortcuts,
sub_clients["explist"],
rpc_clients["schedule"],
rpc_clients["repository"])
rpc_clients["experiment_db"])

d_datasets = datasets.DatasetsDock(win, dock_area, sub_clients["datasets"])
smgr.register(d_datasets)
16 changes: 8 additions & 8 deletions artiq/frontend/artiq_master.py
Original file line number Diff line number Diff line change
@@ -13,7 +13,7 @@
from artiq.master.databases import DeviceDB, DatasetDB
from artiq.master.scheduler import Scheduler
from artiq.master.worker_db import get_last_rid
from artiq.master.repository import FilesystemBackend, GitBackend, Repository
from artiq.master.experiments import FilesystemBackend, GitBackend, ExperimentDB


def get_argparser():
@@ -71,10 +71,10 @@ def main():
repo_backend = GitBackend(args.repository)
else:
repo_backend = FilesystemBackend(args.repository)
repository = Repository(repo_backend, device_db.get_device_db,
log_worker)
atexit.register(repository.close)
repository.scan_async()
experiment_db = ExperimentDB(repo_backend, device_db.get_device_db,
log_worker)
atexit.register(experiment_db.close)
experiment_db.scan_repository_async()

worker_handlers = {
"get_device_db": device_db.get_device_db,
@@ -83,7 +83,7 @@ def main():
"update_dataset": dataset_db.update,
"log": log_worker
}
scheduler = Scheduler(get_last_rid() + 1, worker_handlers, repo_backend)
scheduler = Scheduler(get_last_rid() + 1, worker_handlers, experiment_db)
worker_handlers.update({
"scheduler_submit": scheduler.submit,
"scheduler_delete": scheduler.delete,
@@ -97,7 +97,7 @@ def main():
"master_device_db": device_db,
"master_dataset_db": dataset_db,
"master_schedule": scheduler,
"master_repository": repository
"master_experiment_db": experiment_db
})
loop.run_until_complete(server_control.start(
args.bind, args.port_control))
@@ -107,7 +107,7 @@ def main():
"schedule": scheduler.notifier,
"devices": device_db.data,
"datasets": dataset_db.data,
"explist": repository.explist,
"explist": experiment_db.explist,
"log": log_buffer.data
})
loop.run_until_complete(server_notify.start(
8 changes: 4 additions & 4 deletions artiq/gui/experiments.py
Original file line number Diff line number Diff line change
@@ -301,7 +301,7 @@ def update_log_level(index):
log_level.currentIndexChanged.connect(update_log_level)

repo_rev = QtGui.QLineEdit()
repo_rev.setPlaceholderText("HEAD")
repo_rev.setPlaceholderText("current")
repo_rev_label = QtGui.QLabel("Revision:")
repo_rev_label.setToolTip("Experiment repository revision "
"(commit ID) to use")
@@ -366,11 +366,11 @@ def restore_state(self, state):
class ExperimentManager:
def __init__(self, status_bar, dock_area,
explist_sub, schedule_sub,
schedule_ctl, repository_ctl):
schedule_ctl, experiment_db_ctl):
self.status_bar = status_bar
self.dock_area = dock_area
self.schedule_ctl = schedule_ctl
self.repository_ctl = repository_ctl
self.experiment_db_ctl = experiment_db_ctl

self.submission_scheduling = dict()
self.submission_options = dict()
@@ -497,7 +497,7 @@ def request_inst_term(self, expname):

async def recompute_arginfo(self, expname):
expinfo = self.explist[expname]
description = await self.repository_ctl.examine(expinfo["file"])
description = await self.experiment_db_ctl.examine(expinfo["file"])
return description[expinfo["class_name"]]["arginfo"]

def save_state(self):
4 changes: 2 additions & 2 deletions artiq/gui/explorer.py
Original file line number Diff line number Diff line change
@@ -23,7 +23,7 @@ def __setitem__(self, k, v):

class ExplorerDock(dockarea.Dock):
def __init__(self, status_bar, exp_manager, d_shortcuts,
explist_sub, schedule_ctl, repository_ctl):
explist_sub, schedule_ctl, experiment_db_ctl):
dockarea.Dock.__init__(self, "Explorer", size=(1500, 500))
self.layout.setSpacing(5)
self.layout.setContentsMargins(5, 5, 5, 5)
@@ -86,7 +86,7 @@ def __init__(self, status_bar, exp_manager, d_shortcuts,
scan_repository_action = QtGui.QAction("(Re)scan repository HEAD",
self.el)
def scan_repository():
asyncio.ensure_future(repository_ctl.scan_async())
asyncio.ensure_future(experiment_db_ctl.scan_repository_async())
self.status_bar.showMessage("Requested repository scan")
scan_repository_action.triggered.connect(scan_repository)
self.el.addAction(scan_repository_action)
28 changes: 14 additions & 14 deletions artiq/master/repository.py → artiq/master/experiments.py
Original file line number Diff line number Diff line change
@@ -75,31 +75,31 @@ def _sync_explist(target, source):
target[k] = source[k]


class Repository:
def __init__(self, backend, get_device_db_fn, log_fn):
self.backend = backend
class ExperimentDB:
def __init__(self, repo_backend, get_device_db_fn, log_fn):
self.repo_backend = repo_backend
self.get_device_db_fn = get_device_db_fn
self.log_fn = log_fn

self.cur_rev = self.backend.get_head_rev()
self.backend.request_rev(self.cur_rev)
self.cur_rev = self.repo_backend.get_head_rev()
self.repo_backend.request_rev(self.cur_rev)
self.explist = Notifier(dict())

self._scanning = False

def close(self):
# The object cannot be used anymore after calling this method.
self.backend.release_rev(self.cur_rev)
self.repo_backend.release_rev(self.cur_rev)

async def scan(self, new_cur_rev=None):
async def scan_repository(self, new_cur_rev=None):
if self._scanning:
return
self._scanning = True
try:
if new_cur_rev is None:
new_cur_rev = self.backend.get_head_rev()
wd, _ = self.backend.request_rev(new_cur_rev)
self.backend.release_rev(self.cur_rev)
new_cur_rev = self.repo_backend.get_head_rev()
wd, _ = self.repo_backend.request_rev(new_cur_rev)
self.repo_backend.release_rev(self.cur_rev)
self.cur_rev = new_cur_rev
new_explist = await _scan_experiments(wd, self.get_device_db_fn,
self.log_fn)
@@ -108,13 +108,13 @@ async def scan(self, new_cur_rev=None):
finally:
self._scanning = False

def scan_async(self, new_cur_rev=None):
asyncio.ensure_future(exc_to_warning(self.scan(new_cur_rev)))
def scan_repository_async(self, new_cur_rev=None):
asyncio.ensure_future(exc_to_warning(self.scan_repository(new_cur_rev)))

async def examine(self, filename, use_repository=True):
if use_repository:
revision = self.cur_rev
wd, _ = self.backend.request_rev(revision)
wd, _ = self.repo_backend.request_rev(revision)
filename = os.path.join(wd, filename)
worker = Worker({
"get_device_db": self.get_device_db_fn,
@@ -125,7 +125,7 @@ async def examine(self, filename, use_repository=True):
finally:
await worker.close()
if use_repository:
self.backend.release_rev(revision)
self.repo_backend.release_rev(revision)
return description


21 changes: 11 additions & 10 deletions artiq/master/scheduler.py
Original file line number Diff line number Diff line change
@@ -127,23 +127,24 @@ def get(self):


class RunPool:
def __init__(self, ridc, worker_handlers, notifier, repo_backend):
def __init__(self, ridc, worker_handlers, notifier, experiment_db):
self.runs = dict()
self.state_changed = Condition()

self.ridc = ridc
self.worker_handlers = worker_handlers
self.notifier = notifier
self.repo_backend = repo_backend
self.experiment_db = experiment_db

def submit(self, expid, priority, due_date, flush, pipeline_name):
# mutates expid to insert head repository revision if None.
# called through scheduler.
rid = self.ridc.get()
if "repo_rev" in expid:
if expid["repo_rev"] is None:
expid["repo_rev"] = self.repo_backend.get_head_rev()
wd, repo_msg = self.repo_backend.request_rev(expid["repo_rev"])
expid["repo_rev"] = self.experiment_db.cur_rev
wd, repo_msg = self.experiment_db.repo_backend.request_rev(
expid["repo_rev"])
else:
wd, repo_msg = None, None
run = Run(rid, pipeline_name, wd, expid, priority, due_date, flush,
@@ -159,7 +160,7 @@ async def delete(self, rid):
run = self.runs[rid]
await run.close()
if "repo_rev" in run.expid:
self.repo_backend.release_rev(run.expid["repo_rev"])
self.experiment_db.repo_backend.release_rev(run.expid["repo_rev"])
del self.runs[rid]


@@ -325,8 +326,8 @@ async def _do(self):


class Pipeline:
def __init__(self, ridc, deleter, worker_handlers, notifier, repo_backend):
self.pool = RunPool(ridc, worker_handlers, notifier, repo_backend)
def __init__(self, ridc, deleter, worker_handlers, notifier, experiment_db):
self.pool = RunPool(ridc, worker_handlers, notifier, experiment_db)
self._prepare = PrepareStage(self.pool, deleter.delete)
self._run = RunStage(self.pool, deleter.delete)
self._analyze = AnalyzeStage(self.pool, deleter.delete)
@@ -386,12 +387,12 @@ async def _do(self):


class Scheduler:
def __init__(self, next_rid, worker_handlers, repo_backend):
def __init__(self, next_rid, worker_handlers, experiment_db):
self.notifier = Notifier(dict())

self._pipelines = dict()
self._worker_handlers = worker_handlers
self._repo_backend = repo_backend
self._experiment_db = experiment_db
self._terminated = False

self._ridc = RIDCounter(next_rid)
@@ -422,7 +423,7 @@ def submit(self, pipeline_name, expid, priority, due_date, flush):
logger.debug("creating pipeline '%s'", pipeline_name)
pipeline = Pipeline(self._ridc, self._deleter,
self._worker_handlers, self.notifier,
self._repo_backend)
self._experiment_db)
self._pipelines[pipeline_name] = pipeline
pipeline.start()
return pipeline.pool.submit(expid, priority, due_date, flush, pipeline_name)
2 changes: 1 addition & 1 deletion doc/manual/management_system.rst
Original file line number Diff line number Diff line change
@@ -107,7 +107,7 @@ You may now run the master with the Git support enabled: ::

Push commits containing experiments to the bare repository using e.g. Git over SSH, and the new experiments should automatically appear in the GUI.

.. note:: If you plan to run the ARTIQ system entirely on a single machine, you may also consider using a non-bare repository and the ``post-commit`` hook to trigger repository scans every time you commit changes (locally). The ARTIQ master never uses the repository's working directory, but only what is committed. More precisely, it fetches by default the last (atomically) completed commit at the time of experiment submission and checks it out in a temporary folder (which solves the problem of concurrent repository access).
.. note:: If you plan to run the ARTIQ system entirely on a single machine, you may also consider using a non-bare repository and the ``post-commit`` hook to trigger repository scans every time you commit changes (locally). The ARTIQ master never uses the repository's working directory, but only what is committed. More precisely, when scanning the repository, it fetches the last (atomically) completed commit at that time of repository scan and checks it out in a temporary folder. This commit ID is used by default when subsequently submitting experiments. There is one temporary folder by commit ID currently referenced in the system, so concurrently running experiments from different repository revisions is fully supported by the master.

The GUI always runs experiments from the repository. The command-line client, by default, runs experiment from the raw filesystem (which is useful for iterating rapidly without creating many disorganized commits). If you want to use the repository instead, simply pass the ``-R`` option.