Jscrawler software release added

Jscrawler is a Nodejs app used to crawl a website and build the sitemap xml file.
Sitemaps are downloadable through http server deployed in the SR.

software/jscrawler/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[instance]
+md5sum = 6c17361a49cfc47564063b867aab6e8c
+filename = instance.cfg.in
+
+[template-jscrawler]
+md5sum = 8fa425275120e8ba5c466aff00b48e7b
+filename = instance-jscrawler.cfg.jinja2.in
+
+[template-jscrawler-builder]
+md5sum = c5e8f8b3983d5f572a564b34fa0f7499
+filename = template-jscrawler.builder.sh.in

software/jscrawler/instance-jscrawler-input-schema.json

+{
+  "type": "object",
+  "$schema": "http://json-schema.org/draft-06/schema",
+  "title": "Input Parameters",
+  "properties": {
+    "urls": {
+      "title": "List of website URL to crawl (newline separated).",
+      "description": "List of website URL to crawl (newline separated).",
+      "type": "string",
+      "textarea": true,
+      "default": ""
+    },
+    "crawl-periodicity": {
+      "title": "Number of days periodicity before re-crawl a site.",
+      "description": "Number of days before next crawl of URL will start and regenerate the sitemap. Set 0 to disable re-crawl.",
+      "type": "integer",
+      "default": 0,
+      "enum": [
+        0,
+        1,
+        2,
+        3,
+        4,
+        5,
+        6,
+        7,
+        8,
+        9,
+        10,
+        15,
+        20,
+        30,
+        60
+      ]
+    }
+  }
+}

software/jscrawler/instance-jscrawler-output-schema.json

+{
+  "name": "Output Parameters",
+  "properties": {
+    "url": {
+      "title": "Frontend http server URL",
+      "description": "Frontend http server URL for sitemap download.",
+      "type": "string",
+      "format": "uri"
+    }
+  }
+}

software/jscrawler/instance-jscrawler.cfg.jinja2.in

+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = ${buildout:directory}/etc
+bin = ${buildout:directory}/bin
+srv = ${buildout:directory}/srv
+var = ${buildout:directory}/var
+run = ${:var}/run
+log = ${:var}/log
+scripts = ${:etc}/run
+services = ${:etc}/service
+plugins = ${:etc}/plugin
+ssl = ${:etc}/ssl
+www = ${:srv}/www
+tmp = ${:srv}/tmp
+
+
+#################################
+# httpd service
+#################################
+[gen-certificate]
+recipe = plone.recipe.command
+command = "{{ parameter_dict['openssl'] }}" req -newkey rsa -batch -new -x509 -days 3650 -nodes -keyout "${:ssl_key}" -out "${:ssl_crt}"
+stop-on-error = true
+ssl_crt = ${directory:ssl}/httpd.crt
+ssl_key = ${directory:ssl}/httpd.key
+
+[httpd-wrapper]
+recipe = slapos.cookbook:simplehttpserver
+host = {{ (ipv6 | list)[0] }}
+port = 9083
+base-path = ${directory:www}
+wrapper = ${directory:services}/http-server
+log-file = ${directory:log}/httpd.log
+use-hash-url = false
+cert-file = ${gen-certificate:ssl_crt}
+key-file = ${gen-certificate:ssl_key}
+
+[request-jscrawler-frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = jscawler Frontend
+# XXX We have hardcoded SR URL here.
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config-url = https://[${httpd-wrapper:host}]:${httpd-wrapper:port}
+return = secure_access domain
+
+[jscrawler-frontend-promise]
+<= monitor-promise-base
+module = check_url_available
+name = jscrawler_frontend.py
+config-url = ${request-jscrawler-frontend:connection-secure_access}
+config-check-secure = 1
+
+[logrotate-entry-httpd]
+<= logrotate-entry-base
+name = http-server
+log = ${httpd-wrapper:log-file}
+
+
+[httpd-listen-promise]
+<= monitor-promise-base
+module = check_port_listening
+name = httpd-listen.py
+config-hostname = ${httpd-wrapper:host}
+config-port = ${httpd-wrapper:port}
+
+[jscrawler-wrapper]
+recipe = slapos.cookbook:wrapper
+command-line =
+  {{ parameter_dict['nodejs-location'] }} {{ parameter_dict['jscrawler-location'] }} 
+wrapper-path = ${directory:bin}/jscrawler
+
+[jscrawler-build-wrapper]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_dict['template-jscrawler'] }}
+rendered = ${directory:bin}/jscrawler-build
+extensions = jinja2.ext.do
+mode = 0700
+list = {{ slapparameter_dict.get('urls', "").split("\n") | join('\n  ') }}
+period = {{ slapparameter_dict.get('crawl-periodicity', 0) }}
+context =
+  key public_folder directory:www
+  key tmp_folder directory:tmp
+  key jscrawler_wrapper jscrawler-wrapper:wrapper-path
+  key period :period
+  key url_list :list
+  raw shell_binary {{ bash_executable_location }}
+  raw pid_file ${directory:run}/jscrawler.pid
+
+[cron-entry-build-sitemap]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = jscrawler-build
+frequency = * * * * *
+command = ${jscrawler-build-wrapper:rendered}
+
+
+[publish-connection-information]
+<= monitor-publish
+recipe = slapos.cookbook:publish.serialised
+url = ${request-jscrawler-frontend:connection-secure_access}
+
+[buildout]
+extends = {{ template_monitor }}
+
+parts =
+  publish-connection-information
+  logrotate-entry-httpd
+# crawler cron
+  cron-entry-build-sitemap
+  httpd-wrapper
+  httpd-listen-promise
+  jscrawler-frontend-promise
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true

software/jscrawler/instance.cfg.in

+[buildout]
+parts = switch-softwaretype
+
+# std stuff for slapos instance
+eggs-directory = {{ buildout_egg_directory }}
+develop-eggs-directory = {{ buildout_develop_directory }}
+offline = true
+
+
+[switch-softwaretype]
+recipe = slapos.cookbook:switch-softwaretype
+default  = dynamic-template-jscrawler:rendered
+RootSoftwareInstance = ${:default}
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/${:filename}
+extensions = jinja2.ext.do
+mode = 0644
+extra-context =
+context =
+  key develop_eggs_directory buildout:develop-eggs-directory
+  key buildout_directory buildout:directory
+  key eggs_directory buildout:eggs-directory
+  key ipv4 slap-configuration:ipv4
+  key ipv6 slap-configuration:ipv6
+  key global_ipv4_prefix network-information:global-ipv4-network
+  key slapparameter_dict slap-configuration:configuration
+  key computer_id slap-configuration:computer
+  raw bash_executable_location {{ bash_location }}/bin/dash
+  raw template_monitor {{ template_monitor_cfg }}
+  raw logrotate_cfg {{ logrotate_cfg }}
+  ${:extra-context}
+
+
+[dynamic-template-jscrawler-parameters]
+openssl = {{ openssl_location }}/bin/openssl
+jscrawler-location = {{ jscrawler_location }}/crawler.js
+nodejs-location = {{ nodejs_location }}/bin/node
+template-jscrawler = {{ template_jscrawler_builder }}
+
+[dynamic-template-jscrawler]
+<= jinja2-template-base
+template = {{ template_jscrawler }}
+filename = instance-jscrawler.cfg
+extra-context =
+  section parameter_dict dynamic-template-jscrawler-parameters

software/jscrawler/software.cfg

+[buildout]
+extends =
+  ../../component/bash/buildout.cfg
+  ../../component/openssl/buildout.cfg
+  ../../component/curl/buildout.cfg
+  ../../component/git/buildout.cfg
+  ../../stack/slapos.cfg
+  ../../stack/nodejs.cfg
+  ../../stack/monitor/buildout.cfg
+  ./buildout.hash.cfg
+
+parts =
+  slapos-cookbook
+  nodejs
+  openssl
+  jscrawler-build
+  instance
+
+[nodejs]
+<= nodejs-12.18.3
+
+[jscrawler]
+recipe = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/Mynij/mynij-crawler.git
+revision = ccbdfdc4712c008034b891f081be92b9342c48ac
+git-executable = ${git:location}/bin/git
+
+
+[jscrawler-build]
+recipe = plone.recipe.command
+stop-on-error = true
+command =
+  cd ${jscrawler:location} && PATH=${nodejs:location}/bin:$PATH npm install
+update-command = ${:command}
+
+[download-template]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:filename}
+mode = 0644
+
+[instance]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/instance.cfg
+template = ${:_profile_base_location_}/${:filename}
+mode = 0644
+context =
+    key bash_location bash:location
+    key bin_directory buildout:bin-directory
+    key buildout_egg_directory buildout:eggs-directory
+    key buildout_develop_directory buildout:develop-eggs-directory
+    key buildout_directory buildout:directory
+    key template_monitor_cfg monitor2-template:rendered
+    key logrotate_cfg template-logrotate-base:rendered
+    key jscrawler_location jscrawler:location
+    key nodejs_location nodejs:location
+    key openssl_location openssl:location
+    key template_jscrawler template-jscrawler:target
+    key template_jscrawler_builder template-jscrawler-builder:target
+
+
+[template-jscrawler]
+<= download-template
+output = ${buildout:directory}/instance-jscrawler.cfg.jinja2
+
+[template-jscrawler-builder]
+<= download-template
+output = ${buildout:directory}/template-jscrawler.builder.sh.in

software/jscrawler/software.cfg.json

+{
+  "name": "JSCrawler",
+  "description": "JSCrawler",
+  "serialisation": "xml",
+  "software-type": {
+    "default": {
+      "title": "Default",
+      "description": "JSCrawler",
+      "request": "instance-jscrawler-input-schema.json",
+      "response": "instance-jscrawler-output-schema.json",
+      "index": 0
+    }
+  }
+}

software/jscrawler/template-jscrawler.builder.sh.in

+#!/bin/bash
+
+URLS="{{ url_list }}"
+
+OUTPUT_DIR="{{ public_folder }}"
+TMP_DIR="{{ tmp_folder }}"
+PERIOD="{{ period }}"
+
+if [ -s "{{ pid_file}}" ]; then
+  echo "Crawler process already running with pid `cat {{ pid_file}}`"
+  exit 1
+fi
+
+trap "rm -f -- '{{ pid_file}}'" EXIT
+
+echo $$ > "{{ pid_file}}"
+
+crawl() {
+  {{ jscrawler_wrapper }} -f $TMP_OUTPUT --link $1
+  if [ -s "$2" ]; then
+    mv $2 $3
+  fi
+}
+
+check_crawl() {
+  if [ -z "$PERIOD" ] || [ "$PERIOD" -eq 0 ] ; then
+    echo "Already crawled $1... SKIPPED"
+    return 0;
+  fi
+  sitemap=$3
+  tmp=$2
+  url=$1
+  NOW=$(date +"%s")
+  T=$(stat -c %Y $sitemap)
+  I=$((T+86400*PERIOD))
+  diff=$((NOW-I))
+  if [ "$diff" -gt 0 ]; then
+    crawl $url $tmp $sitemap
+  else
+    echo "Already crawled $url... SKIPPED"
+  fi
+}
+
+for url in `echo $URLS`
+do
+  # Name is the URL domain
+  NAME=$(echo "$url" | cut -d'/' -f3)
+  # Add path is it exists in URL
+  NAME="$NAME$(echo $url | grep -Po '\w\K/\w+[^?]+' | sed -r 's/\/+/_/g')"
+  echo "Checking $NAME..."
+  OUTPUT="$OUTPUT_DIR/$NAME.xml"
+  TMP_OUTPUT="$TMP_DIR/$NAME.xml"
+
+  if [ -s "$OUTPUT" ]; then
+    check_crawl $url $TMP_OUTPUT $OUTPUT
+  else
+    crawl $url $TMP_OUTPUT $OUTPUT
+  fi
+done