Add nayuos software recipe.

This is a SR to build NayuOS binary images.
More info on https://www.nayuos.com and in the README.

Merging this recipe from a standalone repository was a decision made together with @kirr and @vpelletier.

software/nayuos/README.md

+# NayuOS
+
+This is a SlapOS recipe to build NayuOS. It needs to be put in the <code>/srv/slapgrid/\<part\>/srv/runner/project/slapos/software/</code> directory. The created directory is called <code>\<nayuos_build_dirname\></code> in this documentation.
+
+## License
+
+GPL v2 or later
+
+## Requirements
+
+*   sudo and git on the host (for now)
+*   slapuser with sudo rights to execute the cros_sdk scripts (needed to access the chroot environment provided by Chromium OS)
+in /etc/sudoers:
+<pre><code>
+\<slapuser\>       ALL= NOPASSWD: /srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/parts/clone-depot-tools/cros_sdk, /srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/parts/chromiumos/\<release\>/chromite/bin/cros_sdk, /srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/parts/chromiumos/\<release\>/chromite/bootstrap/cros_sdk, /srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/parts/chromiumos/\<release\>/chromite/scripts/cros_sdk.py, /srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/parts/chromiumos/\<release\>/chromite/scripts/cros_sdk.pyc, /srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/parts/chromiumos/\<release\>/chromium/tools/depot_tools/cros_sdk
+</code></pre>
+
+## Input
+In the vifib parameters (softinst\<nb\>.host.vifib.net \> Services \> Parameters):
+
+*   board / ex: peppy, swanky, ... (choosing daisy will accept all licenses for the daisy board build only, in order to use Mali drivers, see [chromium mailing list](https://groups.google.com/a/chromium.org/forum/#!topic/chromium-os-dev/Pf9ZG2itxWM))
+*   branch / ex: release-R46-7390.B
+*   keep_cache / yes|no (choosing "no" saves about 15Go of disk space per board, choosing "yes" will makes next build faster and less expensive in term of needed ressources because of not rebuilding everything)
+
+
+## Output
+The image will be produced in:
+<code>/srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/parts/chromiumos/images/</code>
+and the logs are in:
+<code>/srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/var/log/cros_sources_dl.log</code> and <code>/srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/var/log/cros_build.log</code>
+
+The script that download the sources and build is located in
+<code>/srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/etc/run</code>
+
+## External documents
+
+*   [ <code>repo</code> command reference ](https://source.android.com/source/using-repo.html)
+*   [ NayuOS official website ](https://www.nayuos.org)
+*   [ crouton for chroot ](https://github.com/dnschneid/crouton) ([warning about verified boot](https://github.com/dnschneid/crouton/blob/2a1fc9da380650f47e2bcf37d00962bfb68c4830/installer/main.sh#L517-L536))
+
+## Notes for possible improvements
+*   [ Running virtual machines on your chromebook ](https://www.chromium.org/chromium-os/developer-information-for-chrome-os-devices/running-virtual-machines-on-your-chromebook)
+
+*   to have a more common User Agent (the one of ChromiumOS/NayuOS is quite rare and identifies the user, see [studies of the EFF](https://panopticlick.eff.org/static/browser-uniqueness.pdf)), it seems possible to change the User-Agent flag for guest mode in the getOffTheRecord function, and adding a line (key "kUserAgent" , value "some common user agent" string). Then rebuild Chromium and [add it to NayuOS](https://www.chromium.org/chromium-os/developer-guide#TOC-Making-changes-to-the-Chromium-web-).

software/nayuos/instance.cfg

+[buildout]
+parts =
+  parameters
+  template-full-build-script
+
+# eggs given by software.cfg
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+
+#################################### Tools, sources getter ##################################
+
+[depot-tools]
+# needed for the "repo" command (to download many git repositories)
+recipe = slapos.recipe.build:gitclone
+repository = https://chromium.googlesource.com/chromium/tools/depot_tools.git
+branch = master
+
+[use-depot-tools]
+export-path =
+  export PATH=${depot-tools:location}:"$PATH";
+
+[nayuos-ebuilds]
+recipe = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/nexedi/nayuos-ebuilds.git
+
+############################################################################################
+
+[parameters]
+# for use of the "parameters" XML configuration given from the
+# webrunner interface
+# if needed, .serialized => json (more complex structures are OK, but need another 
+# jinja template for reading the content of the parameters)
+recipe = slapos.cookbook:slapconfiguration
+computer = ${slap_connection:computer_id}
+partition = ${slap_connection:partition_id}
+url = ${slap_connection:server_url}
+key = ${slap_connection:key_file}
+cert = ${slap_connection:cert_file}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+log = ${buildout:directory}/var/log
+run = ${buildout:directory}/etc/run
+cros_location = ${buildout:directory}/parts/chromiumos
+ebuilds_dir = ${nayuos-ebuilds:location}
+scripts_dir = {{ scripts_dir }}
+logo_dir = {{ logo_dir }}
+
+[template-full-build-script]
+# create the instance.cfg file in the buildout directory
+# (use jinja for templating)
+recipe = slapos.recipe.template:jinja2
+template = {{ scripts_dir }}/cros_full_build.in
+rendered = ${directory:run}/cros_full_build
+md5sum = 76942f32e682b36397b41db28297625d
+mode = 0770
+context =
+# for access to the eggs from the instance
+  key instance_log_dir directory:log
+  key cros_location directory:cros_location
+  key depot_tools_export_path_cmd use-depot-tools:export-path
+  key branch parameters:configuration.branch
+  key boards_list parameters:configuration.boards
+  key keep_cache parameters:configuration.keep_cache
+  key ebuilds_dir directory:ebuilds_dir
+  key scripts_dir directory:scripts_dir
+  key logo_dir directory:logo_dir
+  raw nayu_dev_packages net-libs/nodejs net-misc/re6stnet dev-vcs/git dev-python/flask dev-python/virtualenv sys-fs/cryptsetup
\ No newline at end of file

software/nayuos/scripts/cros_full_build.in

+#!/bin/bash
+#XXX: should not depend on the host's bash ^
+
+######################## Download of sources using the "repo" command ########################
+DL_LOG={{ instance_log_dir }}/cros_sources_dl.log
+BRANCH={{ branch }}
+CHROMIUM_OVERLAY={{ cros_location }}/{{ branch }}/src/third_party/chromiumos-overlay
+
+echo "getting Chromium OS sources..." >> $DL_LOG
+{{ depot_tools_export_path_cmd }}
+echo "{{ cros_location }}/{{ branch }}"
+install -d "{{ cros_location }}/{{ branch }}"
+cd "{{ cros_location }}/{{ branch }}"
+
+repo init -u https://chromium.googlesource.com/chromiumos/manifest.git -b {{ branch }} >> $DL_LOG
+
+# in case changes were already made, stash changes to be able to pull
+cd ${CHROMIUM_OVERLAY}
+git stash
+
+repo sync >> $DL_LOG
+
+############################## Prepare chroot environment ###################################
+BUILD_LOG={{ instance_log_dir }}/cros_build.log
+TEST_LOG={{ instance_log_dir }}/tests_nayuos_image.log
+{{ depot_tools_export_path_cmd }}
+cd {{ cros_location }}/{{ branch }}
+
+# create chroot environment (exit on failure)
+cros_sdk --download || { echo "Problem entering chroot or downloading chroot environment. Exiting." ; exit 1 ; }
+
+# compile Python with sqlite support (-> change USE flag) for dev-lang/python
+# it is needed by re6st
+# just need to change -sqlite by sqlite on the right line
+# same for IPv6 support
+PACKAGE_USE_FILE=${CHROMIUM_OVERLAY}/profiles/targets/chromeos/package.use
+line_number=$(sed -n '/dev-lang\/python/=' ${PACKAGE_USE_FILE})
+if [ $line_number ] ; then
+    echo $line_number
+    sed -i "${line_number}s/-sqlite/sqlite/" ${PACKAGE_USE_FILE}
+    if [[ ! $(sed -n ${line_number}p ${PACKAGE_USE_FILE} | grep ipv6) ]]; then
+        sed -i "${line_number}s/$/ ipv6/" ${PACKAGE_USE_FILE}
+    fi
+fi
+
+# add some ebuilds, specific to NayuOS
+for category in $( ls {{ ebuilds_dir }} ); do
+  echo ${category}
+  cp -R {{ ebuilds_dir }}/${category}/* ${CHROMIUM_OVERLAY}/${category}/
+done
+
+install -m 770 "{{ scripts_dir }}/test_nayuos_image" "{{ cros_location }}/{{ branch }}/src/scripts/test_nayuos_image"
+
+### packages management ###
+
+BASE_CHROMEOS_DEV_ROOT_EBUILD=${CHROMIUM_OVERLAY}/chromeos-base/chromeos-dev-root/chromeos-dev-root-0.0.1.ebuild
+VIRTUAL_CHROMEOS_OS_DEV_EBUILD=${CHROMIUM_OVERLAY}/virtual/target-chromium-os-dev/target-chromium-os-dev-1.ebuild
+
+# change the virtual ebuild responsible for installing all packages to add the ones
+# needed for NayuOS
+for package in {{ nayu_dev_packages }} ; do
+  echo $package
+  if [[ $( grep "${package}" "${VIRTUAL_CHROMEOS_OS_DEV_EBUILD}" ) ]] ; then
+    echo "no need to change ${VIRTUAL_CHROMEOS_OS_DEV_EBUILD} file to add ${package}..." >> "${BUILD_LOG}"
+  else
+    printf "\n\nRDEPEND=\"\${RDEPEND}\n          ${package}\"\n">> ${VIRTUAL_CHROMEOS_OS_DEV_EBUILD}
+  fi
+done
+
+# do not install the Upstart init script that starts ssh daemon at boot time
+rm ${CHROMIUM_OVERLAY}/chromeos-base/chromeos-sshd-init/files/openssh-server.conf
+sed -i -n '/src_install/q;p' ${CHROMIUM_OVERLAY}/chromeos-base/openssh-server-init/openssh-server-init-0.0.1.ebuild
+sed -i -n '/src_install/q;p' ${CHROMIUM_OVERLAY}/chromeos-base/chromeos-sshd-init/chromeos-sshd-init-0.0.1.ebuild
+
+# bashrc modifications
+BASH_EBUILD_DIR={{ cros_location }}/{{ branch }}/src/third_party/portage-stable/app-shells/bash
+if [[ $(grep "git --exec-path" "${BASH_EBUILD_DIR}/files/dot-bashrc") ]] ; then
+  echo "git alias already set"
+else
+  echo "# git quickfix for finding right git executables
+if [ -d /usr/local/libexec/git-core/ ] ; then
+    alias git='git --exec-path=/usr/local/libexec/git-core/'
+fi
+
+# git quickfix for finding 'less' and using it as pager
+less_path=\$(which less 2>/dev/null)
+if [ "\${less_path}" ] ; then
+    git config --global core.pager \${less_path}
+fi" >> ${BASH_EBUILD_DIR}/files/dot-bashrc
+fi
+
+# increase the revision number
+# XXX: remove hardcoded revision numbers
+cd ${CHROMIUM_OVERLAY}/chromeos-base/openssh-server-init/
+ln -s openssh-server-init-0.0.1-r24.ebuild openssh-server-init-0.0.1-r25.ebuild
+cd ${CHROMIUM_OVERLAY}/chromeos-base/chromeos-sshd-init/
+ln -s chromeos-sshd-init-0.0.1-r5.ebuild chromeos-sshd-init-0.0.1-r6.ebuild
+cd ${BASH_EBUILD_DIR}
+ln -s bash-4.2_p48-r1.ebuild bash-4.2_p48-r2.ebuild
+
+######################################## Build ##############################################
+BOARDS="{{ boards_list }}"
+KEEP_CACHE="{{ keep_cache }}"
+for board in ${BOARDS} ; do
+  echo ${board}
+  if [ ${board} == daisy ] ; then
+    echo "daisy board: accepting license for Mali drivers..."
+    cros_sdk -- sudo sh -c "cp /etc/make.conf.user /etc/make.conf.user.save"
+    cros_sdk -- sudo sh -c "echo 'ACCEPT_LICENSE=\"*\"' >> /etc/make.conf.user"
+  fi
+
+  # preparing packages (for chroot and image)
+  date >> "${BUILD_LOG}"
+  echo "building packages for a ${board}-flavoured Chromium OS..." >> "${BUILD_LOG}"
+  cros_sdk -- ./build_packages --board=${board} >> "${BUILD_LOG}"
+
+  # change boot pictures
+  cros_sdk -- cros_workon --board=${board} start chromiumos-assets
+  cros_sdk -- cros_workon_make --board=${board} chromiumos-assets
+  cros_sdk -- cros_workon_make --board=${board} chromiumos-assets --test
+  cros_sdk -- cros_workon_make --board=${board} chromiumos-assets --install
+  cp {{ logo_dir }}/* {{ cros_location }}/{{ branch }}/src/platform/chromiumos-assets/images_100_percent/
+  cp {{ logo_dir }}/* {{ cros_location }}/{{ branch }}/src/platform/chromiumos-assets/images_200_percent/
+
+  NAYU_IMAGE_LOCATION=${board}.nayuos.img
+
+  # rebuild packages with boot pictures
+  cros_sdk -- ./build_packages --board=${board} >> "${BUILD_LOG}"
+
+  # NayuOS images
+  date >> "${BUILD_LOG}"
+  echo "building image" >> "${BUILD_LOG}"
+  cros_sdk -- ./build_image --board=${board} dev >> "${BUILD_LOG}" \
+    && cros_sdk -- rm -f $NAYU_IMAGE_LOCATION && cros_sdk -- touch $NAYU_IMAGE_LOCATION \
+    && cros_sdk -- cros flash --board=${board} file://$NAYU_IMAGE_LOCATION >> "${BUILD_LOG}" \
+    && cros_sdk -- ./test_nayuos_image ${board} > "${TEST_LOG}" \
+    || { echo "An error occured while building ${board} NayuOS image. Exiting." ; exit 1 ;}
+
+  # save ~15Go/device but delete cache (next build will be as long)
+  if [ ${KEEP_CACHE,,} == "no" ] ; then
+    cros_sdk -- sudo rm -R /var/cache/chromeos-chrome/chrome-src/src/out_${board}
+  fi
+
+  if [ ${board} == daisy ]; then
+    echo "daisy board: removing accepted license for the next builds..."
+    cros_sdk -- sudo sh -c "mv /etc/make.conf.user.save /etc/make.conf.user"
+  fi
+done
+
+####################################### Post build ##########################################
+# keep only the substring between - as current release number
+RELEASE=$(echo ${BRANCH} | cut -d- -f2)
+DIR_IMAGE_LOCATION={{ cros_location }}/images/${RELEASE}/$(date +'%F')
+install ${DIR_IMAGE_LOCATION} -d
+mv {{ cros_location }}/{{ branch }}/src/scripts/*.img ${DIR_IMAGE_LOCATION}
+
+cd ${DIR_IMAGE_LOCATION}
+for hashfunction in md5sum sha1sum sha256sum sha512sum; do
+  echo ${hashfunction} >> hashes.txt
+  ${hashfunction} *.img >> hashes.txt
+  printf "\n\n" >> hashes.txt
+done
+
+for file in $(ls *.img); do
+  gzip -9 ${file}
+done
+
+exit 0

software/nayuos/scripts/test_nayuos_image

+#!/bin/bash
+
+FAILURE="FAILURE"
+SUCCESS="SUCCESS"
+
+if [ -z "$1" ] ; then
+  echo "Missing board argument. Exiting."
+  exit 1
+fi
+
+BOARD=$1
+MOUNTPOINT="/tmp/${BOARD}"
+ORIGINAL_GRANDENET_SCRIPT=~/trunk/src/third_party/chromiumos-overlay/net-misc/re6stnet/files/grandenet
+GRANDENET_SCRIPT="usr/local/bin/grandenet"
+INIT_SSH_SERVER="etc/init/openssh-server.conf"
+BASHRC="etc/skel/.bashrc"
+EXPECTED_ALIAS="alias git='git --exec-path=/usr/local/libexec/git-core/'"
+VIRTUALENV_BIN="usr/local/bin/virtualenv"
+
+HAS_FAILED=0
+
+function print_result() {
+  test_result=$1
+  message=$2
+
+  printf "\t${test_result}: "
+  printf "${message}\n"
+  if [[ ${test_result} == ${FAILURE} ]] ; then
+      HAS_FAILED=1
+  fi
+}
+
+# MOUNT IMAGE AND GET INFO
+
+install -d ${MOUNTPOINT}
+./mount_gpt_image.sh --safe -f $( ./get_latest_image.sh --board=${BOARD} ) -r ${MOUNTPOINT}
+echo $(ls "${MOUNTPOINT}/usr/local")
+if [[ $(ls "${MOUNTPOINT}/usr/local") ]] ; then
+  my_diff=$(diff ${ORIGINAL_GRANDENET_SCRIPT} "${MOUNTPOINT}/${GRANDENET_SCRIPT}")
+  opensshd_config=$(ls "${MOUNTPOINT}/${INIT_SSH_SERVER}")
+  gitalias=$(grep "${EXPECTED_ALIAS}" "${MOUNTPOINT}/${BASHRC}")
+  virtualenvbin=$(ls "${MOUNTPOINT}/${VIRTUALENV_BIN}")
+else
+  is_empty=1
+fi
+./mount_gpt_image.sh --safe -f $( ./get_latest_image.sh --board=${BOARD} ) -r ${MOUNTPOINT} -u
+rmdir ${MOUNTPOINT}
+
+
+# PRINT RESULTS
+
+echo "* test if /usr/local exists"
+if [[ ${is_empty} == 1 ]] ; then
+  print_result ${FAILURE} "/usr/local is empty."
+else
+  print_result ${SUCCESS} "/usr/local is not empty."
+
+  echo "* test grandenet script existence and content"
+  if [[ ${no_grandenet_script} == 1 ]] ; then
+    print_result ${FAILURE} "grandenet script is missing (no file at ${GRANDENET_SCRIPT})."
+  elif [[ ${my_diff} != "" ]] ; then
+    print_result ${FAILURE} "grandenet scripts differs:\n${my_diff}"
+  else
+    print_result ${SUCCESS} "${GRANDENET_SCRIPT} exists and contains what is expected."
+  fi
+
+  echo "* test openssh server init script absence"
+  if [[ ${opensshd_config} ]] ; then
+    print_result ${FAILURE} "opensshd config exists: ${opensshd_config}"
+  else
+    print_result ${SUCCESS} "opensshd config removed."
+  fi
+
+  echo "* test git quick fix for option --exec-path"
+
+  if [[ ${gitalias} == ""  ]] ; then
+    print_result ${FAILURE} "Expected alias for git command not in ${MOUNTPOINT}/${BASHRC}. Should be: ${EXPECTED_ALIAS}"
+  else
+    print_result ${SUCCESS} "git alias is correct."
+  fi
+
+  echo "* test if virtualenv binary exists"
+
+  if [[ ${virtualenvbin} ]] ; then
+    print_result ${SUCCESS} "virtualenv binary exists."
+  else
+    print_result ${FAILURE} "Expected virtualenv binary not in ${MOUNTPOINT}/${VIRTUALENV_BIN}."
+  fi
+fi
+
+exit ${HAS_FAILED}

software/nayuos/software.cfg

+[buildout]
+extends =
+# basic needs
+  ../../stack/slapos.cfg
+
+parts +=
+# use stack/slapos.cfg
+  slapos-cookbook
+  verify-sudo-exists-on-host
+  template-instance
+  install-eggs-for-the-instance
+
+#add eggs needed by the instance
+find-links += 
+  https://pypi.python.org/packages/source/s/slapos.recipe.build/slapos.recipe.build-0.21.tar.gz
+  https://pypi.python.org/packages/source/p/plone.recipe.command/plone.recipe.command-1.1.zip
+
+versions = versions
+
+[versions]
+plone.recipe.command = 1.1
+slapos.recipe.template = 2.8
+slapos.recipe.build = 0.21
+
+[verify-sudo-exists-on-host]
+recipe = plone.recipe.command
+stop-on-error = true
+# run the same command when installing and on updates
+update-command = ${:command}
+command = sudo -V
+
+[template-instance]
+# create the instance.cfg file in the buildout directory
+# (use jinja for templating)
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance.cfg
+rendered = ${buildout:directory}/instance.cfg
+md5sum = d5a12b1ea814b4bb6792dfebaa655f90
+mode = 0644
+scripts_dir = ${:_profile_base_location_}/scripts
+logo_dir = ${:_profile_base_location_}/logo
+context =
+# for access to the eggs from the instance
+  key eggs_directory buildout:eggs-directory
+  key develop_eggs_directory buildout:develop-eggs-directory
+  key scripts_dir :scripts_dir
+  key logo_dir :logo_dir
+
+[install-eggs-for-the-instance]
+# after installation of the eggs, the recipes
+# will be available (added to sys.path)
+recipe = zc.recipe.egg
+eggs = slapos.recipe.build
+       plone.recipe.command
\ No newline at end of file