Support source_urls ending with .git (#5598)

* support source_urls ending with .git

* crew: clean up

* make sha256sum more readable

* change git_branch to git_tag and allow git hashes (thanks @satmandu)

* support git hashes

* support caching with git sources

* crew: fix compressdoc and fix creating tarball

* lib/const.rb: change envvar to CREW_IS_CACHING

* packages/dash.rb: fix homepage and binaries

* crew: remove (currently) extraneous variable long_verbose

* bin/crew: fix compressdoc

* packages/dash.rb: fix homepages (again)

* bin/crew: change $extract_dir to @extract_dir

* CREW_IS_CACHING => CREW_CACHE_ENABLED

* condense tag + hash checkout logic part 1

* condense tag + hash checkout logic part 2

* Bump version
Co-authored-by: satmandu <satadru@gmail.com>
Co-authored-by: satmandu <satadru@umich.edu>

bin/crew

@@ -111,11 +111,11 @@ String.use_color = args["--color"] || !args["--no-color"]
 if @opt_verbose then
   @fileutils_verbose = true
   @verbose = 'v'
-  @mv_verbose = '-v'
+  @short_verbose = '-v'
 else
   @fileutils_verbose = false
   @verbose = ''
-  @mv_verbose = ''
+  @short_verbose = ''
 end
 
 @opt_src = args["--build-from-source"]
@@ -543,6 +543,11 @@ def download
   url = @pkg.get_url(@device[:architecture])
   source = @pkg.is_source?(@device[:architecture])
 
+  uri = URI.parse url
+  filename = File.basename(uri.path)
+  sha256sum = @pkg.get_sha256(@device[:architecture])
+  @extract_dir = @pkg.get_extract_dir
+
   if !url
     abort "No precompiled binary or source is available for #{@device[:architecture]}.".lightred
   elsif !source
@@ -553,65 +558,98 @@ def download
     puts "No precompiled binary available for your platform, downloading source..."
   end
 
-  uri = URI.parse url
-  filename = File.basename(uri.path)
-  if source
-    sha256sum = @pkg.source_sha256
-  else
-    sha256sum = @pkg.binary_sha256[@device[:architecture]]
-  end
   Dir.chdir CREW_BREW_DIR do
-    if ENV['CREW_CACHE_OPT']
-      FileUtils.mkdir_p CREW_CACHE_DIR, verbose: @fileutils_verbose
-      cachefile = CREW_CACHE_DIR + filename
-      if File.file?(cachefile)
-        if Digest::SHA256.hexdigest( File.read(cachefile) ) == sha256sum then
-          FileUtils.cp cachefile, CREW_BREW_DIR, verbose: @fileutils_verbose
-          puts "Archive found in cache".lightgreen
-          return {source: source, filename: filename}
+
+    case File.basename(filename)
+    # Sources that download with curl
+    when /\.zip$/i, /\.(tar(\.(gz|bz2|xz|lz))?|tgz|tbz|txz)$/i, /\.deb$/i
+      # Recall file from cache if requeseted
+      if CREW_CACHE_ENABLED
+        cachefile = CREW_CACHE_DIR + filename
+        if File.file?(cachefile)
+          if Digest::SHA256.hexdigest( File.read(cachefile) ) == sha256sum then
+            FileUtils.cp cachefile, CREW_BREW_DIR, verbose: @fileutils_verbose
+            puts "Archive found in cache".lightgreen
+            return {source: source, filename: filename}
           else
-          puts 'Cached archive checksum mismatch. :/ Will download.'.lightred
+            puts 'Cached archive checksum mismatch. :/ Will download.'.lightred
+          end
+        puts 'Cannot find cached archive. :/ Will download.'.lightred
         end
       end
-    end
-
-    system "#{CURL} --retry 3 -#{@verbose}#LC - --insecure \'#{url}\' --output #{filename}"
-
-    abort 'Checksum mismatch. :/ Try again.'.lightred unless
-      Digest::SHA256.hexdigest( File.read(filename) ) == sha256sum
-    puts "Archive downloaded".lightgreen
+      # Download file if not cached
+      system "#{CURL} --retry 3 -#{@verbose}#LC - --insecure \'#{url}\' --output #{filename}"
+      abort 'Checksum mismatch. :/ Try again.'.lightred unless
+        Digest::SHA256.hexdigest( File.read(filename) ) == sha256sum
+      puts "Archive downloaded".lightgreen
+      # Stow file in cache if requested
+      if CREW_CACHE_ENABLED
+        puts 'Caching downloaded archive'.lightgreen
+        FileUtils.cp filename, CREW_CACHE_DIR, verbose: @fileutils_verbose
+      end
 
-    if ENV['CREW_CACHE_OPT']
-      puts 'Caching downloaded archive'.lightgreen
-      FileUtils.cp filename, CREW_CACHE_DIR, verbose: @fileutils_verbose
+    # Sources that download with git
+    when /\.git$/i
+      # Recall repository from cache if requested
+      if CREW_CACHE_ENABLED
+        cachefile = CREW_CACHE_DIR + '/' + filename + '.tar.xz'
+        if File.file?(cachefile)
+          if system "sha256sum -c #{cachefile}.sha256"
+            FileUtils.mkdir @extract_dir
+            system "tar x#{@verbose}f #{cachefile} -C #{@extract_dir}"
+            return {source: source, filename: filename}
+          else
+            puts 'Cached repository checksum mismatch. :/ Will download.'.lightred
+          end
+        else
+          puts 'Cannot find cached repository. :/ Will download.'.lightred
+        end
+      end
+      # Download via git
+      Dir.mkdir @extract_dir
+      Dir.chdir @extract_dir do
+        system 'git init'
+        system 'git config advice.detachedHead false'
+        system 'git config init.defaultBranch master'
+        system "git remote add origin #{@pkg.source_url}", exception: true
+        system "git fetch --depth 1 origin #{@pkg.git_hashtag}", exception: true
+        system 'git checkout FETCH_HEAD'
+        system 'git submodule update --init --recursive'
+        puts "Repository downloaded".lightgreen
+      end
+      # Stow file in cache if requested
+      if CREW_CACHE_ENABLED
+        puts 'Caching downloaded archive.'.lightgreen
+          Dir.chdir "#{@extract_dir}" do system "tar c#{@verbose}Jf #{CREW_CACHE_DIR}/#{filename}.tar.xz $(find -mindepth 1 -maxdepth 1 -printf '%P\n')" end
+        system "sha256sum #{CREW_CACHE_DIR}/#{filename}.tar.xz > #{CREW_CACHE_DIR}/#{filename}.tar.xz.sha256"
+      end
     end
   end
   return {source: source, filename: filename}
 end
 
 def unpack (meta)
-  extract_dir = meta[:filename] + '.' + SecureRandom.alphanumeric(8) + '.dir'
   target_dir = nil
   Dir.chdir CREW_BREW_DIR do
-    FileUtils.mkdir_p extract_dir, verbose: @fileutils_verbose
+    FileUtils.mkdir_p @extract_dir, verbose: @fileutils_verbose
     case File.basename meta[:filename]
     when /\.zip$/i
-      puts "Unpacking archive using 'unzip', this may take a while..."
+      puts "Unpacking archive using 'unzip', this may take a while...".lightgreen
       _verbopt = @opt_verbose ? '-v' : '-qq'
-      system 'unzip', _verbopt, '-d', extract_dir, meta[:filename]
+      system 'unzip', _verbopt, '-d', @extract_dir, meta[:filename], exception: true
     when /\.(tar(\.(gz|bz2|xz|lz))?|tgz|tbz|txz)$/i
-      puts "Unpacking archive using 'tar', this may take a while..."
-      FileUtils.mkdir extract_dir unless Dir.exist?(extract_dir)
-      system "tar x#{@verbose}f #{meta[:filename]} -C #{extract_dir}"
+      puts "Unpacking archive using 'tar', this may take a while...".lightgreen
+      FileUtils.mkdir @extract_dir unless Dir.exist?(@extract_dir)
+      system "tar x#{@verbose}f #{meta[:filename]} -C #{@extract_dir}", exception: true
     when /\.deb$/i
-      puts "Unpacking archive using 'ar', this may take a while..."
-      FileUtils.mkdir extract_dir unless Dir.exist?(extract_dir)
-      system "ar -p #{meta[:filename]} data.tar.xz | xz -dc#{@verbose} | tar x#{@verbose} -C #{extract_dir}"
+      puts "Unpacking archive using 'ar', this may take a while...".lightgreen
+      FileUtils.mkdir @extract_dir unless Dir.exist?(@extract_dir)
+      system "ar -p #{meta[:filename]} data.tar.xz | xz -dc#{@verbose} | tar x#{@verbose} -C #{@extract_dir}", exception: true
     end
     if meta[:source] == true
       # Check the number of directories in the archive
-      entries = Dir["#{extract_dir}/*"]
-      entries = Dir[extract_dir] if entries.empty?
+      entries = Dir["#{@extract_dir}/*"]
+      entries = Dir[@extract_dir] if entries.empty?
       if entries.length == 0
         abort "Empty archive: #{meta[:filename]}".lightred
       elsif entries.length == 1 && File.directory?(entries.first)
@@ -619,11 +657,11 @@ def unpack (meta)
         target_dir = entries.first
       else
         # Use `extract_dir` otherwise
-        target_dir = extract_dir
+        target_dir = @extract_dir
       end
     else
       # Use `extract_dir` for binary distribution
-      target_dir = extract_dir
+      target_dir = @extract_dir
     end
   end
   return CREW_BREW_DIR + target_dir
@@ -639,7 +677,7 @@ def build_and_preconfigure (target_dir)
     # https://stackoverflow.com/questions/42963653/libquadmath-la-is-not-a-valid-libtool-archive-when-configuring-openmpi-with-g
     puts 'Rename all *.la files to *.la_tmp'.lightblue
 
-    system "find #{CREW_LIB_PREFIX} -type f -name *.la -print0 | xargs --null -I{} mv #{@mv_verbose} {} {}_tmp"
+    system "find #{CREW_LIB_PREFIX} -type f -name *.la -print0 | xargs --null -I{} mv #{@short_verbose} {} {}_tmp"
 
     @pkg.in_build = true
     @pkg.patch
@@ -654,7 +692,7 @@ def build_and_preconfigure (target_dir)
     # Rename all *.la_tmp back to *.la to avoid
     # cannot access '*.la': No such file or directory
     puts 'Rename all *.la_tmp files back to *.la'.lightblue
-    system "find #{CREW_LIB_PREFIX} -type f -name '*.la_tmp' -exec sh -c 'mv #{@mv_verbose} \"$1\" \"${1%.la_tmp}.la\"' _  {} \\;"
+    system "find #{CREW_LIB_PREFIX} -type f -name '*.la_tmp' -exec sh -c 'mv #{@short_verbose} \"$1\" \"${1%.la_tmp}.la\"' _  {} \\;"
   end
 end
 
@@ -678,7 +716,7 @@ def compress_doc (dir)
 
   if Dir.exist? dir
     system "find #{dir} -type f ! -perm -200 | xargs -r chmod u+w"
-    system "compressdoc --gzip -9#{@verbose} #{dir}"
+    system "compressdoc --gzip -9 #{@short_verbose} #{dir}"
   end
 end
 

lib/const.rb

 # Defines common constants used in different parts of crew
 
-CREW_VERSION = '1.8.5'
+CREW_VERSION = '1.8.6'
 
 ARCH_ACTUAL = `uname -m`.strip
 # This helps with virtualized builds on aarch64 machines
@@ -36,10 +36,13 @@ else
 end
 
 if ENV['CREW_CACHE_DIR'].to_s == ''
-  CREW_CACHE_DIR = HOME + '/.cache/crewcache/'
+  CREW_CACHE_DIR = HOME + '/.cache/crewcache'
 else
   CREW_CACHE_DIR = ENV['CREW_CACHE_DIR']
 end
+FileUtils.mkdir_p CREW_CACHE_DIR unless Dir.exist? CREW_CACHE_DIR
+
+CREW_CACHE_ENABLED = ENV['CREW_CACHE_ENABLED']
 
 CREW_DEST_HOME = CREW_DEST_DIR + HOME
 

lib/package.rb

 require 'package_helpers'
 
 class Package
-  property :description, :homepage, :version, :license, :compatibility, :binary_url, :binary_sha256, :source_url, :source_sha256, :is_fake
+  property :description, :homepage, :version, :license, :compatibility, :binary_url, :binary_sha256, :source_url, :source_sha256, :git_hashtag, :is_fake
 
   class << self
     attr_reader :is_fake
@@ -38,15 +38,27 @@ class Package
   end
 
   def self.get_url (architecture)
-    if !@build_from_source && @binary_url && @binary_url.has_key?(architecture)
+    if !@build_from_source and @binary_url and @binary_url.has_key?(architecture)
       return @binary_url[architecture]
     else
       return @source_url
     end
   end
 
+  def self.get_sha256 (architecture)
+    if !@build_from_source and @binary_sha256 and @binary_sha256.has_key?(architecture)
+      return @binary_sha256[architecture]
+    else
+      return @source_sha256
+    end
+  end
+
+  def self.get_extract_dir
+    name + '.' + Time.now.utc.strftime("%Y%m%d%H%M%S") + '.dir'
+  end
+
   def self.is_binary? (architecture)
-    if !@build_from_source && @binary_url && @binary_url.has_key?(architecture)
+    if !@build_from_source and @binary_url and @binary_url.has_key?(architecture)
       return true
     else
       return false
@@ -54,7 +66,7 @@ class Package
   end
 
   def self.is_source? (architecture)
-    if is_binary?(architecture) || is_fake?
+    if is_binary?(architecture) or is_fake?
       return false
     else
       return true

packages/dash.rb

@@ -2,12 +2,12 @@ require 'package'
 
 class Dash < Package
   description 'The Debian Almquist Shell (dash) is a POSIX-compliant shell derived from ash that executes scripts faster than bash and has fewer library dependencies.'
-  homepage 'https://salsa.debian.org/debian/dash'
+  homepage 'https://salsa.debian.org/debian/dash/'
   version '0.5.11.1'
   license 'BSD'
   compatibility 'all'
-  source_url 'https://salsa.debian.org/debian/dash/-/archive/upstream/0.5.11.1/dash-upstream-0.5.11.1.tar.bz2'
-  source_sha256 '4bb75944bb47fa6d1cf1e8a3ff941f6f1fb23497b553446e9f615d52d07ef1e7'
+  source_url 'https://salsa.debian.org/debian/dash.git'
+  git_tag 'upstream/0.5.11.2'
 
   binary_url ({
     aarch64: 'https://downloads.sourceforge.net/project/chromebrew/armv7l/dash-0.5.11.1-chromeos-armv7l.tar.xz',