Blame view

software/nayuos/scripts/cros_full_build.in 12.8 KB
iv committed
1
#! {{ bash_path }}
iv committed
2

Tomáš Peterka committed
3

iv committed
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
function increase_revision_number() {
  # this increase the revision number an ebuild
  # (except for revision number 9999 which is a value for "live ebuild", for dev revisions)
  # see https://devmanual.gentoo.org/general-concepts/ebuild-revisions/

  if [ -z "$1" ] ; then
    echo "ERROR: No parameter given to function: increase_revision_number."
    echo "Usage: increase_revision_number /path/to/dir/containing/ebuild"
    exit 1
  elif [ -d "$1" ]; then
    DIR="$1"
    CURRENT_FILE=""
    CURRENT_REVISION=0
    # get the biggest revision number
    # XXX: may break if there are many version of an ebuild
    # (eg: bash-4.3_p42-r4.ebuild and bash-3.7-r8.ebuild)
    for FILENAME in $(find "${DIR}" -maxdepth 1 -regextype sed -regex .*-r[0-9]*\.ebuild); do
      echo $FILENAME
      REVISION="$(echo ${FILENAME} | rev | cut -d- -f1 | rev | tr -d [:alpha:] | tr -d [:punct:])"
      echo "${REVISION}"
      if [ "$CURRENT_REVISION" -lt "$REVISION" ] ; then
          CURRENT_REVISION=${REVISION}
          CURRENT_FILE=${FILENAME}
      fi
    done
    if [ "$CURRENT_REVISION" -ne "9999" ]; then
      # increase REVISION
      NEW_REVISION=$((CURRENT_REVISION+1))
      NEW_FILE="$(echo ${CURRENT_FILE} | sed -r 's/-r'${CURRENT_REVISION}'/-r'${NEW_REVISION}'/')"
      echo "changing revision number: ${CURRENT_FILE} -> ${NEW_FILE}"
      mv "${CURRENT_FILE}" "${NEW_FILE}"
    fi
  fi
}

Tomáš Peterka committed
39

Tomáš Peterka committed
40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65
function insert_packages_into_ebuild() {
  # Insert packages (in form of "category/package") into RDEPEND part of ebuild.
  # Optionally increasing version of given ebuild.
  EBUILD=$1
  changed=0
  shift
  while [ ! -z "$1" ]
  do
    package=$1
    if [[ $( grep "${package}" "${EBUILD}" ) ]] ; then
      echo "no need to change ${EBUILD} file to add ${package}..." >> "${BUILD_LOG}"
    else
      printf "\n\nRDEPEND=\"\${RDEPEND}\n          ${package}\"\n">> ${EBUILD}
      changed=1
    fi
    shift
  done;

  if [ $changed -eq 1 ]
  then
    # if we inserted at least one package then increase the version to be sure
    # it gets selected in case of more overlays with the same package
    increase_revision_number $(dirname $EBUILD)
  fi
}

Tomáš Peterka committed
66 67 68 69 70 71 72 73

function latest_ebuild() {
  # find the latest ebuild file (previously we hardcoded the versions)
  # @argument ebuild directory to find the file in
  find $1 -type f -name '*ebuild' | sort -r | head -1
}


iv committed
74
######################## Download of sources using the "repo" command ########################
iv committed
75 76 77
DL_LOG="{{ instance_log_dir }}/cros_sources_dl.log"
BRANCH="{{ branch }}"
CHROMIUM_OVERLAY="{{ cros_location }}/{{ branch }}/src/third_party/chromiumos-overlay"
iv committed
78
PORTAGE_STABLE="{{ cros_location }}/{{ branch }}/src/third_party/portage-stable"
iv committed
79

iv committed
80
{{ export_path_cmd }}
iv committed
81
echo "getting Chromium OS sources..." >> $DL_LOG
iv committed
82 83 84 85
echo "{{ cros_location }}/{{ branch }}"
install -d "{{ cros_location }}/{{ branch }}"
cd "{{ cros_location }}/{{ branch }}"

iv committed
86 87 88 89 90 91 92
# git setup
if ! git config user.name || ! git config user.email ; then
  git config --global user.name "Your Name"
  git config --global user.email "you@example.com" 
fi

repo init -u https://chromium.googlesource.com/chromiumos/manifest.git -b {{ branch }} >> $DL_LOG || { echo "Problem while initiating Chromium OS repo (repo init). PATH is: '${PATH}'. Exiting." ; exit 1 ; }
iv committed
93

iv committed
94
# in case changes were already made, stash changes to be able to pull
iv committed
95 96
cd ${CHROMIUM_OVERLAY}
git stash
iv committed
97 98
cd ${PORTAGE_STABLE}
git stash
iv committed
99

iv committed
100 101
repo sync >> $DL_LOG || { echo "Problem while downloading Chromium OS sources (repo sync). Exiting." ; exit 1 ; }

iv committed
102

iv committed
103
############################## Prepare chroot environment ###################################
iv committed
104
BUILD_LOG={{ instance_log_dir }}/cros_build.log
iv committed
105
TEST_LOG={{ instance_log_dir }}/tests_nayuos_image.log
iv committed
106
cd {{ cros_location }}/{{ branch }}
iv committed
107 108

# create chroot environment (exit on failure)
iv committed
109
cros_sdk --download || { echo "Problem while entering chroot or downloading chroot environment. PATH is: '${PATH}' Exiting." ; exit 1 ; }
iv committed
110

iv committed
111
# add some ebuilds, specific to NayuOS
iv committed
112 113
for category in $( ls {{ ebuilds_dir }} ); do
  echo ${category}
iv committed
114
  cp -R "{{ ebuilds_dir }}/${category}/"* "${CHROMIUM_OVERLAY}/${category}/"
iv committed
115 116
done

iv committed
117 118
install -m 770 "{{ scripts_dir }}/test_nayuos_image" "{{ cros_location }}/{{ branch }}/src/scripts/test_nayuos_image"

iv committed
119 120
### packages management ###

Tomáš Peterka committed
121 122
# make virtual ebuild responsible for installing all packages dependent on NayuOS own packages
insert_packages_into_ebuild \
Tomáš Peterka committed
123
  $(latest_ebuild ${CHROMIUM_OVERLAY}/virtual/target-chromium-os-dev) \
Tomáš Peterka committed
124
  {{ nayu_dev_packages }}
Tomáš Peterka committed
125 126 127

# insert policies into /etc/chromium/policies/recommended with NayuOS customizations
insert_packages_into_ebuild \
Tomáš Peterka committed
128
  $(latest_ebuild ${CHROMIUM_OVERLAY}/chromeos-base/chromeos-dev-root) \
Tomáš Peterka committed
129
  {{ nayu_dev_rootfs_packages }}
iv committed
130

iv committed
131
# do not install the Upstart init script that starts ssh daemon at boot time
Tomáš Peterka committed
132 133
find ${CHROMIUM_OVERLAY}/chromeos-base/chromeos-sshd-init* -name openssh-server.conf -delete

Tomáš Peterka committed
134
# strip away the src_install part because it only sets up booting scripts
Tomáš Peterka committed
135 136
# and increase version in case of other overlays with the same packages
sed -i -n '/src_install/q;p' $(latest_ebuild ${CHROMIUM_OVERLAY}/chromeos-base/openssh-server-init)
iv committed
137
increase_revision_number ${CHROMIUM_OVERLAY}/chromeos-base/openssh-server-init/
Tomáš Peterka committed
138
sed -i -n '/src_install/q;p' $(latest_ebuild ${CHROMIUM_OVERLAY}/chromeos-base/chromeos-sshd-init)
iv committed
139 140
increase_revision_number ${CHROMIUM_OVERLAY}/chromeos-base/chromeos-sshd-init/

Klaus Wölfel committed
141 142 143 144
# increase size of storage quota for chrome guest mode to make big offline apps work
CHROME_EBUILD_DIR="${CHROMIUM_OVERLAY}/chromeos-base/chromeos-chrome"
cp {{ patch_dir }}/0001-chrome-incognito-increase-storage-quota.patch "${CHROME_EBUILD_DIR}/files/"
sed -i 's|^PATCHES=()$|PATCHES=( "${FILESDIR}/0001-chrome-incognito-increase-storage-quota.patch" )|' ${CHROME_EBUILD_DIR}/chromeos-chrome*.ebuild
Tomáš Peterka committed
145

iv committed
146
# bashrc modifications
iv committed
147
BASH_EBUILD_DIR="${PORTAGE_STABLE}/app-shells/bash"
iv committed
148
if ! grep "BEGIN NayuOS configuration" "${BASH_EBUILD_DIR}/files/dot-bashrc" > /dev/null ; then
Tristan Cavelier committed
149 150 151 152
  cat >> "${BASH_EBUILD_DIR}/files/dot-bashrc" <<EOF

# ----- BEGIN NayuOS configuration -----

Tristan Cavelier committed
153 154 155
# use vim as default editor if nano does not exist
which nano &> /dev/null || export EDITOR=vim

Tristan Cavelier committed
156 157 158 159 160 161
# git quickfix for finding right git executables
export GIT_EXEC_PATH=/usr/local/libexec/git-core

# configure .gitconfig once
if [ ! -e ~/.gitconfig ] ; then
  which less &> /dev/null && git config --global core.pager less
iv committed
162 163
fi

iv committed
164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274
# go to ~/Download
[ "\$(pwd)" = / ] && cd ~/Downloads

alias ls='ls -CF --color=auto'
alias l='ls -lF'
alias lh='ls -hAl'
alias la='ls -la'
alias ll='ls -l'

tree() {
  python -Sc 'import sys, os
aa = sys.argv[1:] or ["."]
for a in aa:
  o = len(a.split(os.sep))
  for r, dd, ff in os.walk(a):
    dp = len(r.split(os.sep)) - o + 1
    print "%s- %s" % ((dp - 1) * "  ", os.path.basename(r))
    for f in ff:
      print "%s- %s" % (dp * "  ", f)' "\$@"
}

showzip() {
  [ \$# = 1 ] || { echo "Usage: showzip ZIPFILE" >&2 ; return 1 ; }
  python -Sc 'import sys, zipfile
zfile = zipfile.ZipFile(sys.argv[1])
zfile.printdir()
zfile.close()' "\$@"
}

zip() {
  [ \$# -gt 1 ] || { echo "Usage: zip ZIPFILE FILE..." >&2 ; return 1 ; }
  python -Sc 'import sys, zipfile, os
if os.path.lexists(sys.argv[1]):
  sys.stderr.write("zip: file %r already exists\n" % sys.argv[1])
  sys.exit(1)
zfile = zipfile.ZipFile(sys.argv[1], "w", zipfile.ZIP_DEFLATED)
def arc(arg):
  if os.path.islink(arg):
    sys.stderr.write("zip: symlink %r ignored\n" % arg)
  elif os.path.isdir(arg):
    for r, _, ff in os.walk(arg):
      zfile.write(r)
      for f in ff: arc(os.path.join(r, f))
  else:
    zfile.write(arg)
for arg in sys.argv[2:]: arc(arg)
zfile.close()' "\$@"
}

unzip() {
  [ \$# != 0 ] || { echo "Usage: unzip ZIPFILE [TARGET]" >&2 ; return 1 ; }
  python -Sc 'import sys, zipfile
zfile = zipfile.ZipFile(sys.argv[1])
zfile.extractall(sys.argv[2] if len(sys.argv) > 2 else ".")
zfile.close()' "\$@"
}

encmount() {
  if [ -z \$1 ] ; then
      echo "Please, give a device as parameter, ex: /dev/sda"
      return 1
  fi
  if  [ ! -d /media/removable/key ] ; then
      echo "New dir at /media/removable/key"
      sudo mkdir /media/removable/key
  fi
  echo "Mounting \$1 on /media/removable/key"
  sudo cryptsetup luksOpen \$1 key
  sudo mount -o noexec /dev/mapper/key /media/removable/key
}

__safewrite() {
  local cmd=safewrite
  local usage="Usage: \$cmd DST < stdin

Where DST is the destination file

Copy stdin to DST. Data is flushed regularly to avoid some eventual crash during cache sync.

Example:
  gunzip myimage.img.gz -c | sudosafewrite /dev/sda"

  local KiB=1024
  local MiB=\$((1024 * \$KiB))
  local bs=4096
  local count=\$((50 * \$MiB / \$bs))

  local infostep=50
  local infounit=MiB

  local safewrite_seek=0
  local safewrite_out=
  local safewrite_length=\$infostep

  [ \$# = 2 ] || { echo "\$usage" >&2 ; return 2 ; }
  [ \$1 != - ] || { echo "DST should not be '-'" >&2 ; return 2 ; }

  while [ \$? = 0 ] ; do
    echo "+ copying until \$safewrite_length \$infounit"
    safewrite_out=\$(\$1 bs=\$bs count=\$count seek=\$safewrite_seek of="\$2" 2>&1) || { echo "\$safewrite_out" ; return 1 ; }
    safewrite_length=\$((\$safewrite_length + \$infostep))
    safewrite_seek=\$((\$safewrite_seek + \$count))
    echo "\$safewrite_out"
    ! echo "\$safewrite_out" | grep '(0 B) copied' > /dev/null
  done
  return 0
}
safewrite() { __safewrite dd "\$@" ; }
sudodd() { sudo dd "\$@" ; }
sudosafewrite() { __safewrite sudodd "\$@" ; }

Thomas Gambier committed
275 276 277 278
crew() {
  if [[ ! -x /usr/local/bin/crew ]] ; then
    read -p "Chromebrew is not on this machine; do you want to install it? [Y/n] " -r -n 1 crew_install
    echo ""
Thomas Gambier committed
279
    if [[ ! \$crew_install =~ ^[Nn]$ ]] ; then
Thomas Gambier committed
280 281 282 283 284 285
        echo "Fetching installation file..."
              curl -Ls https://lab.nexedi.com/nexedi/chromebrew/raw/nexedi/install.sh | bash
    else
      return 1
    fi
  else
Thomas Gambier committed
286
    command crew "\$@"
Thomas Gambier committed
287 288 289
  fi
}

Tristan Cavelier committed
290 291
# ----- END NayuOS configuration -----
EOF
iv committed
292
fi
iv committed
293

Tristan Cavelier committed
294
increase_revision_number "${BASH_EBUILD_DIR}"
iv committed
295

iv committed
296 297
######################################## Build ##############################################
BOARDS="{{ boards_list }}"
iv committed
298
KEEP_CACHE="{{ keep_cache }}"
iv committed
299
for board in ${BOARDS} ; do
iv committed
300
  echo ${board}
iv committed
301
  if [ ${board} == daisy ] ; then
iv committed
302
    # XXX: broken by sucessive wrapping
iv committed
303
    echo "daisy board: accepting license for Mali drivers..."
iv committed
304 305
    cros_sdk -- "sudo cp /etc/make.conf.user /etc/make.conf.user.save"
    cros_sdk -- "echo 'ACCEPT_LICENSE=\"*\"' | sudo tee --append /etc/make.conf.user"
iv committed
306
  fi
iv committed
307

iv committed
308
  # preparing packages (for chroot and image)
iv committed
309 310
  date >> "${BUILD_LOG}"
  echo "building packages for a ${board}-flavoured Chromium OS..." >> "${BUILD_LOG}"
Tomáš Peterka committed
311
  cros_sdk --nouse-image -- ./build_packages --board=${board} >> "${BUILD_LOG}"
iv committed
312

iv committed
313 314 315 316 317 318 319 320
  # change boot pictures
  cros_sdk -- cros_workon --board=${board} start chromiumos-assets
  cros_sdk -- cros_workon_make --board=${board} chromiumos-assets
  cros_sdk -- cros_workon_make --board=${board} chromiumos-assets --test
  cros_sdk -- cros_workon_make --board=${board} chromiumos-assets --install
  cp {{ logo_dir }}/* {{ cros_location }}/{{ branch }}/src/platform/chromiumos-assets/images_100_percent/
  cp {{ logo_dir }}/* {{ cros_location }}/{{ branch }}/src/platform/chromiumos-assets/images_200_percent/

Romain Courteaud committed
321
  NAYU_IMAGE_LOCATION=/tmp/${board}.nayuos.img
iv committed
322 323

  # rebuild packages with boot pictures
Thomas Gambier committed
324
  cros_sdk --nouse-image -- ./build_packages --board=${board} >> "${BUILD_LOG}"
iv committed
325

iv committed
326
  # NayuOS images
iv committed
327
  date >> "${BUILD_LOG}"
iv committed
328
  echo "building image" >> "${BUILD_LOG}"
Tomáš Peterka committed
329
  cros_sdk --nouse-image -- ./build_image --board=${board} dev >> "${BUILD_LOG}" \
iv committed
330
    && cros_sdk -- rm -f $NAYU_IMAGE_LOCATION && cros_sdk -- touch $NAYU_IMAGE_LOCATION \
iv committed
331
    && cros_sdk -- cros flash --board=${board} file://$NAYU_IMAGE_LOCATION >> "${BUILD_LOG}" \
iv committed
332
    && cros_sdk -- ./test_nayuos_image ${board} > "${TEST_LOG}" \
iv committed
333
    || { echo "An error occured while building ${board} NayuOS image. Exiting." ; exit 1 ;}
iv committed
334

iv committed
335 336
  # save a lot of space on the server but delete cache and build files
  # (it means that the next build will be as long and use as much resources as this one)
iv committed
337
  if [ ${KEEP_CACHE,,} == "no" ] ; then
iv committed
338 339
    cros_sdk -- sudo rm -R "/var/cache/chromeos-chrome/chrome-src/src/out_${board}"
    cros_sdk -- sudo rm -R "/build/${board}"
iv committed
340 341
  fi

iv committed
342
  if [ ${board} == daisy ]; then
iv committed
343
    # XXX: broken by sucessive wrapping
iv committed
344
    echo "daisy board: removing accepted license for the next builds..."
iv committed
345
    cros_sdk -- "sudo mv /etc/make.conf.user.save /etc/make.conf.user"
iv committed
346 347
  fi
done
iv committed
348 349

####################################### Post build ##########################################
iv committed
350
# keep only the substring between - as current release number
iv committed
351
RELEASE=$(echo ${BRANCH} | cut -d- -f2)
iv committed
352
DIR_IMAGE_LOCATION={{ cros_location }}/images/${RELEASE}/$(date +'%F')
iv committed
353
install ${DIR_IMAGE_LOCATION} -d
Romain Courteaud committed
354
mv {{ cros_location }}/{{ branch }}/chroot/tmp/*.img ${DIR_IMAGE_LOCATION}
iv committed
355 356

cd ${DIR_IMAGE_LOCATION}
iv committed
357
for hashfunction in md5sum sha1sum sha256sum sha512sum; do
iv committed
358
  echo ${hashfunction} >> hashes.txt
iv committed
359
  ${hashfunction} *.img >> hashes.txt
iv committed
360
  printf "\n\n" >> hashes.txt
iv committed
361 362 363
done

for file in $(ls *.img); do
Lorenzo Martinico committed
364
  gzip -9 -f ${file}
iv committed
365 366
done

iv committed
367
exit 0