Skip to content

Commit

Permalink
Merge branch 'master' into new-py2llvm
Browse files Browse the repository at this point in the history
whitequark committed Oct 30, 2015
2 parents b91ffa1 + 62419c0 commit f70f7fb
Showing 114 changed files with 1,110 additions and 1,156 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -13,6 +13,9 @@ doc/manual/_build
/*.egg-info
/.coverage
artiq/test/results
artiq/test/h5types.h5
examples/master/results
examples/master/dataset_db.pyon
examples/sim/dataset_db.pyon
Output/
/lit-test/libartiq_support/libartiq_support.so
43 changes: 27 additions & 16 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -9,29 +9,40 @@ env:
global:
- secure: "DUk/Ihg8KbbzEgPF0qrHqlxU8e8eET9i/BtzNvFddIGX4HP/P2qz0nk3cVkmjuWhqJXSbC22RdKME9qqPzw6fJwJ6dpJ3OR6dDmSd7rewavq+niwxu52PVa+yK8mL4yf1terM7QQ5tIRf+yUL9qGKrZ2xyvEuRit6d4cFep43Ws="
matrix:
- BUILD_SOC=0
- BUILD_SOC=1
before_install:
- mkdir -p $HOME/.mlabs
- if [ $TRAVIS_PULL_REQUEST != false ]; then BUILD_SOC=0; fi
- if [ $BUILD_SOC -ne 0 ]; then ./.travis/get-xilinx.sh; fi
- . ./.travis/get-toolchain.sh
- BUILD_SOC=none
- BUILD_SOC=pipistrello-nist_qc1
- BUILD_SOC=kc705-nist_qc1
- BUILD_SOC=kc705-nist_qc2
install:
- mkdir -p $HOME/.m-labs
- if [ $TRAVIS_PULL_REQUEST != false ]; then BUILD_SOC=none; fi
- if [ $BUILD_SOC != none ]; then ./.travis/get-xilinx.sh; fi
- if [ $BUILD_SOC != none ]; then ./.travis/get-toolchain.sh; fi
- if [ $BUILD_SOC != none ]; then ./.travis/get-misoc.sh; fi
- . ./.travis/get-anaconda.sh
- echo "BUILD_SOC=$BUILD_SOC" >> $HOME/.mlabs/build_settings.sh
- source $HOME/miniconda/bin/activate py35
- conda install -q pip coverage anaconda-client migen cython
- conda install -q pip coverage anaconda-client cython
- pip install coveralls
install:
- conda build --python 3.5 conda/artiq
- conda install -q artiq --use-local
# workaround for https://github.com/conda/conda-build/issues/466
- mkdir -p /home/travis/miniconda/conda-bld/linux-64
- conda index /home/travis/miniconda/conda-bld/linux-64
script:
- coverage run --source=artiq setup.py test
- make -C doc/manual html
- conda build --python 3.5 conda/artiq
- conda install -q --use-local artiq
- |
if [ $BUILD_SOC == none ]; then
PACKAGES="$(conda build --output --python 3.5 conda/artiq) $PACKAGES"
coverage run --source=artiq setup.py test
make -C doc/manual html
else
PACKAGES="$(conda build --output --python 3.5 conda/artiq-$BUILD_SOC) $PACKAGES"
conda build --python 3.5 conda/artiq-$BUILD_SOC
fi
after_success:
- |
if [ "$TRAVIS_BRANCH" == "master" -a $BUILD_SOC -eq 1 ]; then
if [ "$TRAVIS_BRANCH" == "master" -a "$PACKAGES" != "" ]; then
anaconda -q login --hostname $(hostname) --username $binstar_login --password $binstar_password
anaconda -q upload --user $binstar_login --channel dev --force $HOME/miniconda/conda-bld/linux-64/artiq-*.tar.bz2
anaconda -q upload --user $binstar_login --channel dev $PACKAGES
anaconda -q logout
fi
- coveralls
1 change: 1 addition & 0 deletions .travis/get-anaconda.sh
Original file line number Diff line number Diff line change
@@ -10,4 +10,5 @@ conda update -q conda
conda info -a
conda install conda-build jinja2
conda create -q -n py35 python=$TRAVIS_PYTHON_VERSION
conda config --add channels https://conda.anaconda.org/m-labs/channel/main
conda config --add channels https://conda.anaconda.org/m-labs/channel/dev
4 changes: 4 additions & 0 deletions .travis/get-misoc.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
#!/bin/sh

git clone --recursive https://github.com/m-labs/misoc $HOME/misoc
echo "export MSCDIR=$HOME/misoc" >> $HOME/.m-labs/build_settings.sh
18 changes: 2 additions & 16 deletions .travis/get-toolchain.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
#!/bin/sh

packages="http://us.archive.ubuntu.com/ubuntu/pool/universe/i/iverilog/iverilog_0.9.7-1_amd64.deb"
archives="http://fehu.whitequark.org/files/llvm-or1k.tbz2"

mkdir -p packages

@@ -12,18 +11,5 @@ do
dpkg -x $pkg_name packages
done

for a in $archives
do
wget $a
(cd packages && tar xf ../$(basename $a))
done

export PATH=$PWD/packages/usr/local/llvm-or1k/bin:$PWD/packages/usr/local/bin:$PWD/packages/usr/bin:$PATH
export LD_LIBRARY_PATH=$PWD/packages/usr/lib/x86_64-linux-gnu:$PWD/packages/usr/local/x86_64-unknown-linux-gnu/or1k-elf/lib:$LD_LIBRARY_PATH

echo "export LD_LIBRARY_PATH=$PWD/packages/usr/lib/x86_64-linux-gnu:$PWD/packages/usr/local/x86_64-unknown-linux-gnu/or1k-elf/lib:\$LD_LIBRARY_PATH" >> $HOME/.mlabs/build_settings.sh
echo "export PATH=$PWD/packages/usr/local/llvm-or1k/bin:$PWD/packages/usr/local/bin:$PWD/packages/usr/bin:\$PATH" >> $HOME/.mlabs/build_settings.sh

or1k-linux-as --version
llc --version
clang --version
echo "export LD_LIBRARY_PATH=$PWD/packages/usr/lib/x86_64-linux-gnu" >> $HOME/.m-labs/build_settings.sh
echo "export PATH=$PWD/packages/usr/bin:\$PATH" >> $HOME/.m-labs/build_settings.sh
2 changes: 1 addition & 1 deletion .travis/get-xilinx.sh
Original file line number Diff line number Diff line change
@@ -30,7 +30,7 @@ git clone https://github.com/fallen/impersonate_macaddress
make -C impersonate_macaddress
# Tell mibuild where Xilinx toolchains are installed
# and feed it the mac address corresponding to the license
cat >> $HOME/.mlabs/build_settings.sh << EOF
cat >> $HOME/.m-labs/build_settings.sh << EOF
MISOC_EXTRA_VIVADO_CMDLINE="-Ob vivado_path $HOME/Xilinx/Vivado"
MISOC_EXTRA_ISE_CMDLINE="-Ob ise_path $HOME/opt/Xilinx/"
export MACADDR=$macaddress
6 changes: 6 additions & 0 deletions artiq/devices/pdq2/driver.py
Original file line number Diff line number Diff line change
@@ -139,6 +139,12 @@ def __init__(self, url=None, dev=None, num_boards=3):
self.num_channels = self.num_dacs * self.num_boards
self.channels = [Channel() for i in range(self.num_channels)]

def get_num_boards(self):
return self.num_boards

def get_num_channels(self):
return self.num_channels

def close(self):
self.dev.close()
del self.dev
6 changes: 6 additions & 0 deletions artiq/frontend/artiq_client.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#!/usr/bin/env python3

import argparse
import logging
import time
import asyncio
import sys
@@ -51,6 +52,10 @@ def get_argparser():
"(defaults to head, ignored without -R)")
parser_add.add_argument("-c", "--class-name", default=None,
help="name of the class to run")
parser_add.add_argument("-v", "--verbose", default=0, action="count",
help="increase logging level of the experiment")
parser_add.add_argument("-q", "--quiet", default=0, action="count",
help="decrease logging level of the experiment")
parser_add.add_argument("file",
help="file containing the experiment to run")
parser_add.add_argument("arguments", nargs="*",
@@ -110,6 +115,7 @@ def _action_submit(remote, args):
sys.exit(1)

expid = {
"log_level": logging.WARNING + args.quiet*10 - args.verbose*10,
"file": args.file,
"class_name": args.class_name,
"arguments": arguments,
112 changes: 85 additions & 27 deletions artiq/frontend/artiq_ctlmgr.py
Original file line number Diff line number Diff line change
@@ -5,39 +5,22 @@
import argparse
import os
import logging
import subprocess
import shlex
import socket
import platform

from artiq.protocols.sync_struct import Subscriber
from artiq.protocols.pc_rpc import AsyncioClient, Server
from artiq.tools import verbosity_args, init_logger
from artiq.protocols.logging import (LogForwarder,
parse_log_message, log_with_name,
SourceFilter)
from artiq.tools import TaskObject, Condition


logger = logging.getLogger(__name__)


def get_argparser():
parser = argparse.ArgumentParser(description="ARTIQ controller manager")
verbosity_args(parser)
parser.add_argument(
"-s", "--server", default="::1",
help="hostname or IP of the master to connect to")
parser.add_argument(
"--port", default=3250, type=int,
help="TCP port to use to connect to the master")
parser.add_argument(
"--retry-master", default=5.0, type=float,
help="retry timer for reconnecting to master")
parser.add_argument(
"--bind", default="::1",
help="hostname or IP address to bind to")
parser.add_argument(
"--bind-port", default=3249, type=int,
help="TCP port to listen to for control (default: %(default)d)")
return parser


class Controller:
def __init__(self, name, ddb_entry):
self.name = name
@@ -96,14 +79,36 @@ async def _wait_and_ping(self):
else:
break

async def forward_logs(self, stream):
source = "controller({})".format(self.name)
while True:
try:
entry = (await stream.readline())
if not entry:
break
entry = entry[:-1]
level, name, message = parse_log_message(entry.decode())
log_with_name(name, level, message, extra={"source": source})
except:
logger.debug("exception in log forwarding", exc_info=True)
break
logger.debug("stopped log forwarding of stream %s of %s",
stream, self.name)


async def launcher(self):
try:
while True:
logger.info("Starting controller %s with command: %s",
self.name, self.command)
try:
self.process = await asyncio.create_subprocess_exec(
*shlex.split(self.command))
*shlex.split(self.command),
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
asyncio.ensure_future(self.forward_logs(
self.process.stdout))
asyncio.ensure_future(self.forward_logs(
self.process.stderr))
await self._wait_and_ping()
except FileNotFoundError:
logger.warning("Controller %s failed to start", self.name)
@@ -129,14 +134,20 @@ async def _terminate(self):
except:
logger.warning("Controller %s did not respond to terminate "
"command, killing", self.name)
self.process.kill()
try:
self.process.kill()
except ProcessLookupError:
pass
try:
await asyncio.wait_for(self.process.wait(),
self.term_timeout)
except:
logger.warning("Controller %s failed to exit, killing",
self.name)
self.process.kill()
try:
self.process.kill()
except ProcessLookupError:
pass
await self.process.wait()
logger.debug("Controller %s terminated", self.name)

@@ -252,9 +263,48 @@ def retry_now(self, k):
self.controller_db.current_controllers.active[k].retry_now.notify()


def get_argparser():
parser = argparse.ArgumentParser(description="ARTIQ controller manager")

group = parser.add_argument_group("verbosity")
group.add_argument("-v", "--verbose", default=0, action="count",
help="increase logging level of the manager process")
group.add_argument("-q", "--quiet", default=0, action="count",
help="decrease logging level of the manager process")

parser.add_argument(
"-s", "--server", default="::1",
help="hostname or IP of the master to connect to")
parser.add_argument(
"--port-notify", default=3250, type=int,
help="TCP port to connect to for notifications")
parser.add_argument(
"--port-logging", default=1066, type=int,
help="TCP port to connect to for logging")
parser.add_argument(
"--retry-master", default=5.0, type=float,
help="retry timer for reconnecting to master")
parser.add_argument(
"--bind", default="::1",
help="hostname or IP address to bind to")
parser.add_argument(
"--bind-port", default=3249, type=int,
help="TCP port to listen to for control (default: %(default)d)")
return parser


def main():
args = get_argparser().parse_args()
init_logger(args)

root_logger = logging.getLogger()
root_logger.setLevel(logging.NOTSET)
source_adder = SourceFilter(logging.WARNING + args.quiet*10 - args.verbose*10,
"ctlmgr({})".format(platform.node()))
console_handler = logging.StreamHandler()
console_handler.setFormatter(logging.Formatter(
"%(levelname)s:%(source)s:%(name)s:%(message)s"))
console_handler.addFilter(source_adder)
root_logger.addHandler(console_handler)

if os.name == "nt":
loop = asyncio.ProactorEventLoop()
@@ -263,7 +313,15 @@ def main():
loop = asyncio.get_event_loop()
atexit.register(lambda: loop.close())

ctlmgr = ControllerManager(args.server, args.port, args.retry_master)
logfwd = LogForwarder(args.server, args.port_logging,
args.retry_master)
logfwd.addFilter(source_adder)
root_logger.addHandler(logfwd)
logfwd.start()
atexit.register(lambda: loop.run_until_complete(logfwd.stop()))

ctlmgr = ControllerManager(args.server, args.port_notify,
args.retry_master)
ctlmgr.start()
atexit.register(lambda: loop.run_until_complete(ctlmgr.stop()))

16 changes: 14 additions & 2 deletions artiq/frontend/artiq_flash.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,16 @@
#!/bin/bash
#!/usr/bin/env python
# conda-build requires all scripts to have a python shebang.
# see https://github.com/conda/conda-build/blob/6921f067a/conda_build/noarch_python.py#L36-L38

def run(script):
import sys, tempfile, subprocess
file = tempfile.NamedTemporaryFile(mode='w+t', suffix='sh')
file.write(script)
file.flush()
subprocess.run(["/bin/bash", file.name] + sys.argv[1:])
file.close()

run("""
# exit on error
set -e
# print commands
@@ -72,7 +83,7 @@ do
echo ""
echo "To flash everything, do not use any of the -b|-B|-r option."
echo ""
echo "usage: $0 [-b] [-B] [-r] [-h] [-m nist_qc1|nist_qc2] [-t kc705|pipistrello] [-d path] [-f path]"
echo "usage: artiq_flash.sh [-b] [-B] [-r] [-h] [-m nist_qc1|nist_qc2] [-t kc705|pipistrello] [-d path] [-f path]"
echo "-b Flash bitstream"
echo "-B Flash BIOS"
echo "-r Flash ARTIQ runtime"
@@ -193,3 +204,4 @@ then
fi
echo "Done."
xc3sprog -v -c $CABLE -R > /dev/null 2>&1
""")
1 change: 1 addition & 0 deletions artiq/frontend/artiq_gui.py
Original file line number Diff line number Diff line change
@@ -116,6 +116,7 @@ def main():
atexit.register(lambda: loop.run_until_complete(d_schedule.sub_close()))

d_log = LogDock()
smgr.register(d_log)
loop.run_until_complete(d_log.sub_connect(
args.server, args.port_notify))
atexit.register(lambda: loop.run_until_complete(d_log.sub_close()))
Loading

0 comments on commit f70f7fb

Please sign in to comment.