flash_storage: refactor + unit tests + artiq_coreconfig.py CLI + doc

fallen · sbourdeauducq · commit 4bf7875b87f9 · 2015-05-27T18:06:12.000+08:00
diff --git a/artiq/coredevice/comm_generic.py b/artiq/coredevice/comm_generic.py
@@ -20,7 +20,12 @@ class _H2DMsgType(Enum):
     RUN_KERNEL = 5
 
     RPC_REPLY = 6
-    
+
+    FLASH_READ_REQUEST = 7
+    FLASH_WRITE_REQUEST = 8
+    FLASH_ERASE_REQUEST = 9
+    FLASH_REMOVE_REQUEST = 10
+
 
 class _D2HMsgType(Enum):
     LOG_REPLY = 1
@@ -37,6 +42,11 @@ class _D2HMsgType(Enum):
 
     RPC_REQUEST = 10
 
+    FLASH_READ_REPLY = 11
+    FLASH_WRITE_REPLY = 12
+    FLASH_OK_REPLY = 13
+    FLASH_ERROR_REPLY = 14
+
 
 class UnsupportedDevice(Exception):
     pass
@@ -122,6 +132,44 @@ def run(self, kname, reset_now):
         self.write(bytes(kname, "ascii"))
         logger.debug("running kernel: %s", kname)
 
+    def flash_storage_read(self, key):
+        self._write_header(9+len(key), _H2DMsgType.FLASH_READ_REQUEST)
+        self.write(key)
+        length, ty = self._read_header()
+        if ty != _D2HMsgType.FLASH_READ_REPLY:
+            raise IOError("Incorrect reply from device: {}".format(ty))
+        value = self.read(length - 9)
+        return value
+
+    def flash_storage_write(self, key, value):
+        self._write_header(9+len(key)+1+len(value),
+                           _H2DMsgType.FLASH_WRITE_REQUEST)
+        self.write(key)
+        self.write(b"\x00")
+        self.write(value)
+        _, ty = self._read_header()
+        if ty != _D2HMsgType.FLASH_WRITE_REPLY:
+            if ty == _D2HMsgType.FLASH_ERROR_REPLY:
+                raise IOError("Invalid key: not a null-terminated string")
+            else:
+                raise IOError("Incorrect reply from device: {}".format(ty))
+        ret = self.read(1)
+        if ret != b"\x01":
+            raise IOError("Flash storage is full")
+
+    def flash_storage_erase(self):
+        self._write_header(9, _H2DMsgType.FLASH_ERASE_REQUEST)
+        _, ty = self._read_header()
+        if ty != _D2HMsgType.FLASH_OK_REPLY:
+            raise IOError("Incorrect reply from device: {}".format(ty))
+
+    def flash_storage_remove(self, key):
+        self._write_header(9+len(key), _H2DMsgType.FLASH_REMOVE_REQUEST)
+        self.write(key)
+        _, ty = self._read_header()
+        if ty != _D2HMsgType.FLASH_OK_REPLY:
+            raise IOError("Incorrect reply from device: {}".format(ty))
+
     def _receive_rpc_value(self, type_tag):
         if type_tag == "n":
             return None
diff --git a/artiq/frontend/artiq_coreconfig.py b/artiq/frontend/artiq_coreconfig.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+
+import argparse
+
+from artiq.master.worker_db import create_device
+from artiq.protocols.file_db import FlatFileDB
+
+
+def to_bytes(string):
+    return bytes(string, encoding="ascii")
+
+
+def get_argparser():
+    parser = argparse.ArgumentParser(description="ARTIQ core device config "
+                                                 "remote access")
+    parser.add_argument("-r", "--read", type=to_bytes,
+                        help="read key from core device config")
+    parser.add_argument("-w", "--write", nargs=2, action="append", default=[],
+                        metavar=("KEY", "STRING"), type=to_bytes,
+                        help="write key-value records to core device config")
+    parser.add_argument("-f", "--write-file", nargs=2, action="append",
+                        type=to_bytes, default=[], metavar=("KEY", "FILENAME"),
+                        help="write the content of a file into core device "
+                             "config")
+    parser.add_argument("-e", "--erase", action="store_true",
+                        help="erase core device config")
+    parser.add_argument("-d", "--delete", action="append", default=[],
+                        type=to_bytes,
+                        help="delete key from core device config")
+    parser.add_argument("--ddb", default="ddb.pyon",
+                        help="device database file")
+    return parser
+
+
+def main():
+    args = get_argparser().parse_args()
+    ddb = FlatFileDB(args.ddb)
+    comm = create_device(ddb.request("comm"), None)
+
+    if args.read:
+        value = comm.flash_storage_read(args.read)
+        if not value:
+            print("Key {} does not exist".format(args.read))
+        else:
+            print(value)
+    elif args.erase:
+            comm.flash_storage_erase()
+    elif args.delete:
+        for key in args.delete:
+            comm.flash_storage_remove(key)
+    else:
+        if args.write:
+            for key, value in args.write:
+                comm.flash_storage_write(key, value)
+        if args.write_file:
+            for key, filename in args.write_file:
+                with open(filename, "rb") as fi:
+                    comm.flash_storage_write(key, fi.read())
+
+if __name__ == "__main__":
+    main()
diff --git a/artiq/frontend/artiq_flash.sh b/artiq/frontend/artiq_flash.sh
@@ -7,7 +7,7 @@ ARTIQ_PREFIX=$(python3 -c "import artiq; print(artiq.__path__[0])")
 # Default is kc705
 BOARD=kc705
 
-while getopts "bBrht:d:" opt
+while getopts "bBrht:d:f:" opt
 do
 	case $opt in
 		b)
@@ -19,6 +19,15 @@ do
 		r)
 			FLASH_RUNTIME=1
 			;;
+		f)
+			if [ -f $OPTARG ]
+			then
+				FILENAME=$OPTARG
+			else
+				echo "You specified a non-existing file to flash: $OPTARG"
+				exit 1
+			fi
+			;;
 		t)
 			if [ "$OPTARG" == "kc705" ]
 			then
@@ -52,6 +61,7 @@ do
 			echo "-r  Flash ARTIQ runtime"
 			echo "-h  Show this help message"
 			echo "-t  Target (kc705, pipistrello, default is: kc705)"
+			echo "-f  Flash storage image generated with artiq_mkfs"
 			echo "-d  Directory containing the binaries to be flashed"
 			exit 1
 			;;
@@ -95,6 +105,7 @@ then
 	PROXY=bscan_spi_kc705.bit
 	BIOS_ADDR=0xaf0000
 	RUNTIME_ADDR=0xb00000
+	FS_ADDR=0xb40000
 	if [ -z "$BIN_PREFIX" ]; then BIN_PREFIX=$ARTIQ_PREFIX/binaries/kc705; fi
 	search_for_proxy $PROXY
 elif [ "$BOARD" == "pipistrello" ]
@@ -105,12 +116,13 @@ then
 	PROXY=bscan_spi_lx45_csg324.bit
 	BIOS_ADDR=0x170000
 	RUNTIME_ADDR=0x180000
+	FS_ADDR=0x1c0000
 	if [ -z "$BIN_PREFIX" ]; then BIN_PREFIX=$ARTIQ_PREFIX/binaries/pipistrello; fi
 	search_for_proxy $PROXY
 fi
 
 # Check if neither of -b|-B|-r have been used
-if [ -z "$FLASH_RUNTIME" -a -z "$FLASH_BIOS" -a -z "$FLASH_BITSTREAM" ]
+if [ -z "$FLASH_RUNTIME" -a -z "$FLASH_BIOS" -a -z "$FLASH_BITSTREAM" -a -z "$FILENAME" ]
 then
 	FLASH_RUNTIME=1
 	FLASH_BIOS=1
@@ -132,6 +144,12 @@ then
 fi
 set -e
 
+if [ ! -z "$FILENAME" ]
+then
+	echo "Flashing file $FILENAME at address $FS_ADDR"
+	xc3sprog -v -c $CABLE -I$PROXY_PATH/$PROXY $FILENAME:w:$FS_ADDR:BIN
+fi
+
 if [ "${FLASH_BITSTREAM}" == "1" ]
 then
 	echo "Flashing FPGA bitstream..."
diff --git a/artiq/frontend/artiq_mkfs.py b/artiq/frontend/artiq_mkfs.py
@@ -20,16 +20,13 @@ def get_argparser():
 
 
 def write_record(f, key, value):
+    key_size = len(key) + 1
+    value_size = len(value)
+    record_size = key_size + value_size + 4
+    f.write(struct.pack(">l", record_size))
     f.write(key.encode())
     f.write(b"\x00")
-    key_size = len(key) + 1
-    if key_size % 4:
-        f.write(bytes(4 - (key_size % 4)))
-    f.write(struct.pack(">l", len(value)))
     f.write(value)
-    value_size = len(value)
-    if value_size % 4:
-        f.write(bytes(4 - (value_size % 4)))
 
 
 def write_end_marker(f):
diff --git a/artiq/master/worker_db.py b/artiq/master/worker_db.py
@@ -65,7 +65,7 @@ def write_hdf5(self, f):
         result_dict_to_hdf5(f, self.data.read)
 
 
-def _create_device(desc, dbh):
+def create_device(desc, dbh):
     ty = desc["type"]
     if ty == "local":
         module = importlib.import_module(desc["module"])
@@ -105,7 +105,7 @@ def get_device(self, name):
             while isinstance(desc, str):
                 # alias
                 desc = self.ddb.request(desc)
-            dev = _create_device(desc, self)
+            dev = create_device(desc, self)
             self.active_devices[name] = dev
             return dev
 
diff --git a/doc/manual/core_device_flash_storage.rst b/doc/manual/core_device_flash_storage.rst
@@ -0,0 +1,14 @@
+.. _core-device-flash-storage:
+
+Core device flash storage
+=========================
+
+The core device contains some flash space that can be used to store
+some configuration data.
+
+This storage area is used to store the core device MAC address, IP address and even the idle kernel.
+
+The flash storage area is one sector (64 kB) large and is organized as a list
+of key-value records.
+
+This flash storage space can be accessed by using the artiq_coreconfig.py :ref:`core-device-configuration-tool`.
diff --git a/doc/manual/index.rst b/doc/manual/index.rst
@@ -15,6 +15,7 @@ Contents:
     core_drivers_reference
     protocols_reference
     ndsp_reference
+    core_device_flash_storage
     utilities
     fpga_board_ports
     default_network_ports
diff --git a/doc/manual/installing.rst b/doc/manual/installing.rst
@@ -124,6 +124,50 @@ These steps are required to generate bitstream (``.bit``) files, build the MiSoC
 
 The communication parameters are 115200 8-N-1.
 
+* Set the MAC and IP address in the :ref:`core device configuration flash storage <core-device-flash-storage>`:
+
+    * You can either set it by generating a flash storage image and then flash it: ::
+
+        $ ~/artiq-dev/artiq/frontend/artiq_mkfs.py flash_storage.img -s mac xx:xx:xx:xx:xx:xx -s ip xx.xx.xx.xx
+        $ ~/artiq-dev/artiq/frontend/artiq_flash.sh -f flash_storage.img
+
+    * Or you can set it via the runtime test mode command line
+
+        * Boot the board.
+
+        * Quickly run flterm (in ``path/to/misoc/tools``) to access the serial console.
+
+        * If you weren't quick enough to see anything in the serial console, press the reset button.
+
+        * Wait for "Press 't' to enter test mode..." to appear and hit the ``t`` key.
+
+        * Enter the following commands (which will erase the flash storage content).
+
+            ::
+
+                test> fserase
+                test> fswrite ip xx.xx.xx.xx
+                test> fswrite mac xx:xx:xx:xx:xx:xx
+
+        * Then reboot.
+
+        You should see something like this in the serial console: ::
+
+            ~/dev/misoc$ ./tools/flterm --port /dev/ttyUSB1
+            [FLTERM] Starting...
+
+            MiSoC BIOS   http://m-labs.hk
+            (c) Copyright 2007-2014 Sebastien Bourdeauducq
+            [...]
+            Press 't' to enter test mode...
+            Entering test mode.
+            test> fserase
+            test> fswrite ip 192.168.10.2
+            test> fswrite mac 11:22:33:44:55:66
+
+.. note:: The reset button of the KC705 board is the "CPU_RST" labeled button.
+.. warning:: Both those instructions will result in the flash storage being wiped out. However you can use the test mode to change the IP/MAC without erasing everything if you skip the "fserase" command.
+
 Installing the host-side software
 ---------------------------------
 
diff --git a/doc/manual/utilities.rst b/doc/manual/utilities.rst
@@ -92,3 +92,58 @@ This tool compiles key/value pairs into a binary image suitable for flashing int
 .. argparse::
    :ref: artiq.frontend.artiq_mkfs.get_argparser
    :prog: artiq_mkfs
+
+.. _core-device-configuration-tool:
+
+Core device configuration tool
+------------------------------
+
+The artiq_coreconfig tool allows to read, write and remove key-value records from the :ref:`core-device-flash-storage`.
+
+It also allows to erase the entire flash storage area.
+
+To use this tool, you need to specify a ``ddb.pyon`` DDB file which contains a ``comm`` device (an example is provided in ``artiq/examples/master/ddb.pyon``).
+This tells the tool how to connect to the core device (via serial or via TCP) and with which parameters (baudrate, serial device, IP address, TCP port).
+When not specified, the artiq_coreconfig tool will assume that there is a file named ``ddb.pyon`` in the current directory.
+
+
+To read the record whose key is ``mac``::
+
+    $ artiq_coreconfig -r mac
+
+To write the value ``test_value`` in the key ``my_key``::
+
+    $ artiq_coreconfig -w my_key test_value
+    $ artiq_coreconfig -r my_key
+    b'test_value'
+
+You can also write entire files in a record using the ``-f`` parameter::
+
+    $ echo "this_is_a_test" > my_filename
+    $ artiq_coreconfig -f my_key my_filename
+    $ artiq_coreconfig -r my_key
+    b'this_is_a_test\n'
+
+You can write several records at once::
+
+    $ artiq_coreconfig -w key1 value1 -f key2 filename -w key3 value3
+
+To remove the previously written key ``my_key``::
+
+    $ artiq_coreconfig -d my_key
+
+To erase the entire flash storage area::
+
+    $ artiq_coreconfig -e
+
+You don't need to remove a record in order to change its value, just overwrite
+it::
+
+    $ artiq_coreconfig -w my_key some_value
+    $ artiq_coreconfig -w my_key some_other_value
+    $ artiq_coreconfig -r my_key
+    b'some_other_value'
+
+.. argparse::
+   :ref: artiq.frontend.artiq_coreconfig.get_argparser
+   :prog: artiq_coreconfig
diff --git a/soc/runtime/flash_storage.c b/soc/runtime/flash_storage.c
diff --git a/soc/runtime/flash_storage.h b/soc/runtime/flash_storage.h
diff --git a/soc/runtime/session.c b/soc/runtime/session.c
diff --git a/soc/runtime/test_mode.c b/soc/runtime/test_mode.c
