Merge branch 'master' into new-py2llvm

m-labs · Oct 30, 2015 · f70f7fb · f70f7fb
2 parents b91ffa1 + 62419c0
commit f70f7fb
Showing 114 changed files with 1,110 additions and 1,156 deletions.
diff --git a/.gitignore b/.gitignore
@@ -13,6 +13,9 @@ doc/manual/_build
 /*.egg-info
 /.coverage
 artiq/test/results
+artiq/test/h5types.h5
 examples/master/results
+examples/master/dataset_db.pyon
+examples/sim/dataset_db.pyon
 Output/
 /lit-test/libartiq_support/libartiq_support.so
diff --git a/.travis.yml b/.travis.yml
@@ -9,29 +9,40 @@ env:
   global:
     - secure: "DUk/Ihg8KbbzEgPF0qrHqlxU8e8eET9i/BtzNvFddIGX4HP/P2qz0nk3cVkmjuWhqJXSbC22RdKME9qqPzw6fJwJ6dpJ3OR6dDmSd7rewavq+niwxu52PVa+yK8mL4yf1terM7QQ5tIRf+yUL9qGKrZ2xyvEuRit6d4cFep43Ws="
   matrix:
-    - BUILD_SOC=0
-    - BUILD_SOC=1
-before_install:
-  - mkdir -p $HOME/.mlabs
-  - if [ $TRAVIS_PULL_REQUEST != false ]; then BUILD_SOC=0; fi
-  - if [ $BUILD_SOC -ne 0 ]; then ./.travis/get-xilinx.sh; fi
-  - . ./.travis/get-toolchain.sh
+    - BUILD_SOC=none
+    - BUILD_SOC=pipistrello-nist_qc1
+    - BUILD_SOC=kc705-nist_qc1
+    - BUILD_SOC=kc705-nist_qc2
+install:
+  - mkdir -p $HOME/.m-labs
+  - if [ $TRAVIS_PULL_REQUEST != false ]; then BUILD_SOC=none; fi
+  - if [ $BUILD_SOC != none ]; then ./.travis/get-xilinx.sh; fi
+  - if [ $BUILD_SOC != none ]; then ./.travis/get-toolchain.sh; fi
+  - if [ $BUILD_SOC != none ]; then ./.travis/get-misoc.sh; fi
   - . ./.travis/get-anaconda.sh
-  - echo "BUILD_SOC=$BUILD_SOC" >> $HOME/.mlabs/build_settings.sh
   - source $HOME/miniconda/bin/activate py35
-  - conda install -q pip coverage anaconda-client migen cython
+  - conda install -q pip coverage anaconda-client cython
   - pip install coveralls
-install:
-  - conda build --python 3.5 conda/artiq
-  - conda install -q artiq --use-local
+  # workaround for https://github.com/conda/conda-build/issues/466
+  - mkdir -p /home/travis/miniconda/conda-bld/linux-64
+  - conda index /home/travis/miniconda/conda-bld/linux-64
 script:
-  - coverage run --source=artiq setup.py test
-  - make -C doc/manual html
+  - conda build --python 3.5 conda/artiq
+  - conda install -q --use-local artiq
+  - |
+      if [ $BUILD_SOC == none ]; then
+        PACKAGES="$(conda build --output --python 3.5 conda/artiq) $PACKAGES"
+        coverage run --source=artiq setup.py test
+        make -C doc/manual html
+      else
+        PACKAGES="$(conda build --output --python 3.5 conda/artiq-$BUILD_SOC) $PACKAGES"
+        conda build --python 3.5 conda/artiq-$BUILD_SOC
+      fi
 after_success:
   - |
-      if [ "$TRAVIS_BRANCH" == "master" -a $BUILD_SOC -eq 1 ]; then
+      if [ "$TRAVIS_BRANCH" == "master" -a "$PACKAGES" != "" ]; then
         anaconda -q login --hostname $(hostname) --username $binstar_login --password $binstar_password
-        anaconda -q upload --user $binstar_login --channel dev --force $HOME/miniconda/conda-bld/linux-64/artiq-*.tar.bz2
+        anaconda -q upload --user $binstar_login --channel dev $PACKAGES
         anaconda -q logout
       fi
   - coveralls

diff --git a/.travis/get-anaconda.sh b/.travis/get-anaconda.sh
@@ -10,4 +10,5 @@ conda update -q conda
 conda info -a
 conda install conda-build jinja2
 conda create -q -n py35 python=$TRAVIS_PYTHON_VERSION
+conda config --add channels https://conda.anaconda.org/m-labs/channel/main
 conda config --add channels https://conda.anaconda.org/m-labs/channel/dev
diff --git a/.travis/get-misoc.sh b/.travis/get-misoc.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+git clone --recursive https://github.com/m-labs/misoc $HOME/misoc
+echo "export MSCDIR=$HOME/misoc" >> $HOME/.m-labs/build_settings.sh
diff --git a/.travis/get-toolchain.sh b/.travis/get-toolchain.sh
@@ -1,7 +1,6 @@
 #!/bin/sh
 
 packages="http://us.archive.ubuntu.com/ubuntu/pool/universe/i/iverilog/iverilog_0.9.7-1_amd64.deb"
-archives="http://fehu.whitequark.org/files/llvm-or1k.tbz2"
 
 mkdir -p packages
 
@@ -12,18 +11,5 @@ do
 	dpkg -x $pkg_name packages
 done
 
-for a in $archives
-do
-  wget $a
-  (cd packages && tar xf ../$(basename $a))
-done
-
-export PATH=$PWD/packages/usr/local/llvm-or1k/bin:$PWD/packages/usr/local/bin:$PWD/packages/usr/bin:$PATH
-export LD_LIBRARY_PATH=$PWD/packages/usr/lib/x86_64-linux-gnu:$PWD/packages/usr/local/x86_64-unknown-linux-gnu/or1k-elf/lib:$LD_LIBRARY_PATH
-
-echo "export LD_LIBRARY_PATH=$PWD/packages/usr/lib/x86_64-linux-gnu:$PWD/packages/usr/local/x86_64-unknown-linux-gnu/or1k-elf/lib:\$LD_LIBRARY_PATH" >> $HOME/.mlabs/build_settings.sh
-echo "export PATH=$PWD/packages/usr/local/llvm-or1k/bin:$PWD/packages/usr/local/bin:$PWD/packages/usr/bin:\$PATH" >> $HOME/.mlabs/build_settings.sh
-
-or1k-linux-as --version
-llc --version
-clang --version
+echo "export LD_LIBRARY_PATH=$PWD/packages/usr/lib/x86_64-linux-gnu" >> $HOME/.m-labs/build_settings.sh
+echo "export PATH=$PWD/packages/usr/bin:\$PATH" >> $HOME/.m-labs/build_settings.sh
diff --git a/.travis/get-xilinx.sh b/.travis/get-xilinx.sh
@@ -30,7 +30,7 @@ git clone https://github.com/fallen/impersonate_macaddress
 make -C impersonate_macaddress
 # Tell mibuild where Xilinx toolchains are installed
 # and feed it the mac address corresponding to the license
-cat >> $HOME/.mlabs/build_settings.sh << EOF
+cat >> $HOME/.m-labs/build_settings.sh << EOF
 MISOC_EXTRA_VIVADO_CMDLINE="-Ob vivado_path $HOME/Xilinx/Vivado"
 MISOC_EXTRA_ISE_CMDLINE="-Ob ise_path $HOME/opt/Xilinx/"
 export MACADDR=$macaddress

diff --git a/artiq/devices/pdq2/driver.py b/artiq/devices/pdq2/driver.py
@@ -139,6 +139,12 @@ def __init__(self, url=None, dev=None, num_boards=3):
         self.num_channels = self.num_dacs * self.num_boards
         self.channels = [Channel() for i in range(self.num_channels)]
 
+    def get_num_boards(self):
+        return self.num_boards
+
+    def get_num_channels(self):
+        return self.num_channels
+
     def close(self):
         self.dev.close()
         del self.dev

diff --git a/artiq/frontend/artiq_client.py b/artiq/frontend/artiq_client.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import argparse
+import logging
 import time
 import asyncio
 import sys
@@ -51,6 +52,10 @@ def get_argparser():
                                  "(defaults to head, ignored without -R)")
     parser_add.add_argument("-c", "--class-name", default=None,
                             help="name of the class to run")
+    parser_add.add_argument("-v", "--verbose", default=0, action="count",
+                            help="increase logging level of the experiment")
+    parser_add.add_argument("-q", "--quiet", default=0, action="count",
+                            help="decrease logging level of the experiment")
     parser_add.add_argument("file",
                             help="file containing the experiment to run")
     parser_add.add_argument("arguments", nargs="*",
@@ -110,6 +115,7 @@ def _action_submit(remote, args):
         sys.exit(1)
 
     expid = {
+        "log_level": logging.WARNING + args.quiet*10 - args.verbose*10,
         "file": args.file,
         "class_name": args.class_name,
         "arguments": arguments,

diff --git a/artiq/frontend/artiq_ctlmgr.py b/artiq/frontend/artiq_ctlmgr.py
@@ -5,39 +5,22 @@
 import argparse
 import os
 import logging
+import subprocess
 import shlex
 import socket
+import platform
 
 from artiq.protocols.sync_struct import Subscriber
 from artiq.protocols.pc_rpc import AsyncioClient, Server
-from artiq.tools import verbosity_args, init_logger
+from artiq.protocols.logging import (LogForwarder,
+                                     parse_log_message, log_with_name,
+                                     SourceFilter)
 from artiq.tools import TaskObject, Condition
 
 
 logger = logging.getLogger(__name__)
 
 
-def get_argparser():
-    parser = argparse.ArgumentParser(description="ARTIQ controller manager")
-    verbosity_args(parser)
-    parser.add_argument(
-        "-s", "--server", default="::1",
-        help="hostname or IP of the master to connect to")
-    parser.add_argument(
-        "--port", default=3250, type=int,
-        help="TCP port to use to connect to the master")
-    parser.add_argument(
-        "--retry-master", default=5.0, type=float,
-        help="retry timer for reconnecting to master")
-    parser.add_argument(
-        "--bind", default="::1",
-        help="hostname or IP address to bind to")
-    parser.add_argument(
-        "--bind-port", default=3249, type=int,
-        help="TCP port to listen to for control (default: %(default)d)")
-    return parser
-
-
 class Controller:
     def __init__(self, name, ddb_entry):
         self.name = name
@@ -96,14 +79,36 @@ async def _wait_and_ping(self):
             else:
                 break
 
+    async def forward_logs(self, stream):
+        source = "controller({})".format(self.name)
+        while True:
+            try:
+                entry = (await stream.readline())
+                if not entry:
+                    break
+                entry = entry[:-1]
+                level, name, message = parse_log_message(entry.decode())
+                log_with_name(name, level, message, extra={"source": source})
+            except:
+                logger.debug("exception in log forwarding", exc_info=True)
+                break
+        logger.debug("stopped log forwarding of stream %s of %s",
+            stream, self.name)
+
+
     async def launcher(self):
         try:
             while True:
                 logger.info("Starting controller %s with command: %s",
                             self.name, self.command)
                 try:
                     self.process = await asyncio.create_subprocess_exec(
-                        *shlex.split(self.command))
+                        *shlex.split(self.command),
+                        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                    asyncio.ensure_future(self.forward_logs(
+                        self.process.stdout))
+                    asyncio.ensure_future(self.forward_logs(
+                        self.process.stderr))
                     await self._wait_and_ping()
                 except FileNotFoundError:
                     logger.warning("Controller %s failed to start", self.name)
@@ -129,14 +134,20 @@ async def _terminate(self):
             except:
                 logger.warning("Controller %s did not respond to terminate "
                                "command, killing", self.name)
-                self.process.kill()
+                try:
+                    self.process.kill()
+                except ProcessLookupError:
+                    pass
             try:
                 await asyncio.wait_for(self.process.wait(),
                                        self.term_timeout)
             except:
                 logger.warning("Controller %s failed to exit, killing",
                                self.name)
-                self.process.kill()
+                try:
+                    self.process.kill()
+                except ProcessLookupError:
+                    pass
                 await self.process.wait()
         logger.debug("Controller %s terminated", self.name)
 
@@ -252,9 +263,48 @@ def retry_now(self, k):
         self.controller_db.current_controllers.active[k].retry_now.notify()
 
 
+def get_argparser():
+    parser = argparse.ArgumentParser(description="ARTIQ controller manager")
+
+    group = parser.add_argument_group("verbosity")
+    group.add_argument("-v", "--verbose", default=0, action="count",
+                       help="increase logging level of the manager process")
+    group.add_argument("-q", "--quiet", default=0, action="count",
+                       help="decrease logging level of the manager process")
+
+    parser.add_argument(
+        "-s", "--server", default="::1",
+        help="hostname or IP of the master to connect to")
+    parser.add_argument(
+        "--port-notify", default=3250, type=int,
+        help="TCP port to connect to for notifications")
+    parser.add_argument(
+        "--port-logging", default=1066, type=int,
+        help="TCP port to connect to for logging")
+    parser.add_argument(
+        "--retry-master", default=5.0, type=float,
+        help="retry timer for reconnecting to master")
+    parser.add_argument(
+        "--bind", default="::1",
+        help="hostname or IP address to bind to")
+    parser.add_argument(
+        "--bind-port", default=3249, type=int,
+        help="TCP port to listen to for control (default: %(default)d)")
+    return parser
+
+
 def main():
     args = get_argparser().parse_args()
-    init_logger(args)
+
+    root_logger = logging.getLogger()
+    root_logger.setLevel(logging.NOTSET)
+    source_adder = SourceFilter(logging.WARNING + args.quiet*10 - args.verbose*10,
+                                "ctlmgr({})".format(platform.node()))
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(logging.Formatter(
+        "%(levelname)s:%(source)s:%(name)s:%(message)s"))
+    console_handler.addFilter(source_adder)
+    root_logger.addHandler(console_handler)
 
     if os.name == "nt":
         loop = asyncio.ProactorEventLoop()
@@ -263,7 +313,15 @@ def main():
         loop = asyncio.get_event_loop()
     atexit.register(lambda: loop.close())
 
-    ctlmgr = ControllerManager(args.server, args.port, args.retry_master)
+    logfwd = LogForwarder(args.server, args.port_logging,
+                          args.retry_master)
+    logfwd.addFilter(source_adder)
+    root_logger.addHandler(logfwd)
+    logfwd.start()
+    atexit.register(lambda: loop.run_until_complete(logfwd.stop()))
+
+    ctlmgr = ControllerManager(args.server, args.port_notify,
+                               args.retry_master)
     ctlmgr.start()
     atexit.register(lambda: loop.run_until_complete(ctlmgr.stop()))
 

diff --git a/artiq/frontend/artiq_flash.sh b/artiq/frontend/artiq_flash.sh
@@ -1,5 +1,16 @@
-#!/bin/bash
+#!/usr/bin/env python
+# conda-build requires all scripts to have a python shebang.
+# see https://github.com/conda/conda-build/blob/6921f067a/conda_build/noarch_python.py#L36-L38
 
+def run(script):
+		import sys, tempfile, subprocess
+		file = tempfile.NamedTemporaryFile(mode='w+t', suffix='sh')
+		file.write(script)
+		file.flush()
+		subprocess.run(["/bin/bash", file.name] + sys.argv[1:])
+		file.close()
+
+run("""
 # exit on error
 set -e
 # print commands
@@ -72,7 +83,7 @@ do
 			echo ""
 			echo "To flash everything, do not use any of the -b|-B|-r option."
 			echo ""
-			echo "usage: $0 [-b] [-B] [-r] [-h] [-m nist_qc1|nist_qc2] [-t kc705|pipistrello] [-d path] [-f path]"
+			echo "usage: artiq_flash.sh [-b] [-B] [-r] [-h] [-m nist_qc1|nist_qc2] [-t kc705|pipistrello] [-d path] [-f path]"
 			echo "-b  Flash bitstream"
 			echo "-B  Flash BIOS"
 			echo "-r  Flash ARTIQ runtime"
@@ -193,3 +204,4 @@ then
 fi
 echo "Done."
 xc3sprog -v -c $CABLE -R > /dev/null 2>&1
+""")
diff --git a/artiq/frontend/artiq_gui.py b/artiq/frontend/artiq_gui.py
@@ -116,6 +116,7 @@ def main():
     atexit.register(lambda: loop.run_until_complete(d_schedule.sub_close()))
 
     d_log = LogDock()
+    smgr.register(d_log)
     loop.run_until_complete(d_log.sub_connect(
         args.server, args.port_notify))
     atexit.register(lambda: loop.run_until_complete(d_log.sub_close()))