#!/usr/bin/python3

from datetime import datetime
from optparse import OptionParser
import os
import os.path
import re
import shutil
import subprocess
import sys
import time
import urllib.request
import xml.dom.minidom
import json
import tempfile
import select
import zstandard
import tarfile

def CheckCall(args, cwd=None, quiet=False, env=None):
  with open(os.devnull, "w") as devnull:
    p = subprocess.Popen(args, cwd=cwd, env=env,
                         stdout = devnull if quiet == True else None,
                         stderr = devnull if quiet == True else None)
    r = p.wait()
    if r != 0: raise subprocess.CalledProcessError(r, args)

def CheckOutput(args, cwd=None):
  p = subprocess.Popen(args, cwd=cwd, stdout=subprocess.PIPE, universal_newlines=True)
  r = p.wait()
  if r != 0: raise subprocess.CalledProcessError(r, args)
  return p.stdout.read()

def check_external_dependencies():
  for tool in ['hg', 'cargo', 'tar']:
    try:
      CheckCall([tool, '--version'], quiet=True)
    except OSError:
      print('Error: missing external tool: %s' % tool)
      sys.exit(1)

def ensure_cache(repo, cache):
  dest = os.path.join(cache, os.path.basename(repo))
  if os.path.isdir(dest):

    try:
      CheckCall(['hg', 'summary'], cwd=dest, quiet=True)
      print('Cache location %s exists, using it' % dest)
      CheckCall(['hg', 'pull', repo], cwd=dest)
      CheckCall(['hg', 'update'], cwd=dest)
      return
    except:
      pass

  if not os.path.isdir(cache):
    os.makedirs(cache)

  print('Creating cache location %s' % dest)
  CheckCall(['hg', 'clone', repo, dest])

def do_checkout(source, dest, rev=None):
  dest = os.path.abspath(dest)
  dest_parent = os.path.dirname(dest)
  if dest_parent != '' and not os.path.isdir(dest_parent):
    os.makedirs(dest_parent)

  CheckCall(['hg', 'clone', source, dest])

  args = ['hg', 'update']
  if rev != None:
    args.append('-r')
    args.append(rev)
  CheckCall(args, cwd=dest)

def checkout_source(repo, cache, dest, rev=None):
  print('*** Checking out source from %s%s ***' % (repo, ' using cache from %s' % cache if cache != None else ''))
  local_source = None
  if cache != None:
    ensure_cache(repo, cache)
    local_source = os.path.join(cache, os.path.basename(repo))
  source = repo if local_source == None else local_source
  do_checkout(source, dest, rev=rev)

def get_setting(settings, name, default=None):
  return settings[name] if name in settings else default

class ScopedTmpdir:
  def __enter__(self):
    self._tmpdir = tempfile.mkdtemp()
    return self._tmpdir

  def __exit__(self, type, value, traceback):
    shutil.rmtree(self._tmpdir)

class ScopedWorkingDirectory:
  def __init__(self, dir):
    self._wd = os.path.abspath(dir)

  def __enter__(self):
    self._saved_wd = os.getcwd()
    if not os.path.isdir(self._wd):
      os.makedirs(self._wd)

    os.chdir(self._wd)

  def __exit__(self, type, value, traceback):
    os.chdir(self._saved_wd)

class ScopedRename:
  def __init__(self, src, dest):
    self._src = src
    self._dest = dest

  def __enter__(self):
    os.rename(self._src, self._dest)

  def __exit__(self, type, value, traceback):
    os.rename(self._dest, self._src)

class TarballCreator(OptionParser):
  def __init__(self):
    OptionParser.__init__(self, 'usage: %prog [options]')

    self.add_option('-r', '--repo', dest='repo', help='The remote repository from which to pull the main source')
    self.add_option('-s', '--sub-repo', dest='subrepo', help='The remote repository from which to pull the embedding source')
    self.add_option('-k', '--comm-l10n-repo', dest='comml10nrepo', help='The remote repository from which to pull the comm l10n source')
    self.add_option('--sub-repo-dir', dest='subrepodir', help='The subdirectory in which to save the embedding source')
    self.add_option('-c', '--cache', dest='cache', help='A local cache of the remote repositories')
    self.add_option('-l', '--l10n-base-repo', dest='l10nbase', help='The base directory of the remote repositories to pull l10n data from')
    self.add_option('-v', '--version', dest='version', help='Release version to base the checkout on')
    self.add_option('--build', dest='build', help='Release build to base the checkout on (must be used with --version)')
    self.add_option('-n', '--name', dest='name', help='The package name')
    self.add_option('-b', '--basename', dest='basename', help='The package basename')
    self.add_option('-a', '--application', dest='application', help='The application to build')

  def vendor_cbindgen(self, dest):
    # cbindgen is not available in Debian/Ubuntu yet (https://bugs.debian.org/908312)
    print('*** Vendoring cbindgen and its dependencies ***')
    with ScopedTmpdir() as tmpdir:
      with ScopedWorkingDirectory(tmpdir):
        CheckCall(['cargo', 'new', 'vendored-cbindgen', '--vcs', 'none'])
        with ScopedWorkingDirectory('vendored-cbindgen'):
          with open('Cargo.toml', 'a+') as fd:
            fd.write('cbindgen = "=0.26.0"')
          CheckCall(['cargo', 'vendor'])
          with ScopedWorkingDirectory('vendor/cbindgen'):
            os.makedirs('.cargo')
            with open('.cargo/config', 'w') as fd:
              fd.write(CheckOutput(['cargo', 'vendor']))
            with open('Cargo.toml', 'a+') as fd:
              fd.write('\n[workspace]')
        shutil.copytree('vendored-cbindgen/vendor/cbindgen', dest)

  def vendor_dump_syms(self, dest):
    # dump_syms is not available in Debian/Ubuntu yet
    print('*** Vendoring dump_syms and its dependencies ***')
    with ScopedTmpdir() as tmpdir:
      with ScopedWorkingDirectory(tmpdir):
        CheckCall(['cargo', 'new', 'vendored-dump_syms', '--vcs', 'none'])
        with ScopedWorkingDirectory('vendored-dump_syms'):
          with open('Cargo.toml', 'a+') as fd:
            fd.write('dump_syms = { git = "https://github.com/mozilla/dump_syms.git", rev = "c558ad1824b247e147dcc45ec105588883e379fb" }')
          CheckCall(['cargo', 'vendor'])
          with ScopedWorkingDirectory('vendor/dump_syms'):
            os.makedirs('.cargo')
            with open('.cargo/config', 'w') as fd:
              fd.write(CheckOutput(['cargo', 'vendor']))
            with open('Cargo.toml', 'a+') as fd:
              fd.write('\n[workspace]')
        shutil.copytree('vendored-dump_syms/vendor/dump_syms', dest)

  def run(self):
    (options, args) = self.parse_args()

    if options.repo == None:
      self.error('Must specify a remote repository')

    if options.subrepo != None and options.subrepodir == None:
      self.error('Must specify a directory')

    if options.name == None:
      self.error('Must specify a package name')

    if options.application == None:
      self.error('Must specify an application')

    if options.build != None and options.version == None:
      self.error('--build must be used with --version')

    if options.cache != None and not os.path.isabs(options.cache):
      options.cache = os.path.join(os.getcwd(), options.cache)

    settings = None
    with open('debian/config/tarball.conf', 'r') as fd:
      settings = json.load(fd)

    cache = options.cache
    version = options.version
    build = options.build
    application = options.application
    l10nbase = options.l10nbase
    comml10nrepo = options.comml10nrepo
    name = options.name
    basename = options.basename
    if basename == None:
      basename = name

    main_rev = None
    sub_rev = None
    buildid = None

    if version != None:
      if build == None:
        build = 1
      print('*** Determing revisions to use for checkouts ***')
      main_info_url = ('https://ftp.mozilla.org/pub/%s/candidates/%s-candidates/build%s/linux-x86_64/en-US/%s-%s.txt'
                       % (basename, version, build, basename, version))
      u = urllib.request.urlopen(main_info_url)
      for line in u.readlines():
        line = line.strip().decode('utf-8')
        if line.isdigit() and buildid == None:
          buildid = line
        if not line.startswith('https://'):
          continue
        r = re.match('(.*)/rev/(.*)', line)
        if r == None:
          print("Badly formatted file '%s'" % main_info_url, file=sys.stderr)
          sys.exit(1)
        repo = r.group(1)
        if repo == options.repo:
          if main_rev == None:
            main_rev = r.group(2)
            print('Revision to be used for main checkout: %s' % main_rev)
        elif options.subrepo and repo == options.subrepo:
          if sub_rev == None:
            sub_rev = r.group(2)
            print('Revision to be used for sub checkout: %s' % sub_rev)
        else:
          print("Unrecognized url '%s' in file '%s'" % (repo, main_info_url), file=sys.stderr)

      if main_rev == None:
        print('Failed to determine revision for main checkout', file=sys.stderr)
        sys.exit(1)
      if options.subrepo != None and sub_rev == None:
        print('Failed to determine revision for sub repo checkout', file=sys.stderr)
        sys.exit(1)
      if buildid == None:
        buildid = datetime.now().strftime('%Y%m%d%H%M%S')
      print('Build ID: %s' % buildid)

    with ScopedTmpdir() as tmpdir:
      print('*** Using temporary directory %s ***' % tmpdir)
      orig_cwd = os.getcwd()
      with ScopedWorkingDirectory(os.path.join(tmpdir, name)):

        checkout_source(options.repo, cache, '', rev=main_rev)
        if options.subrepo:
          checkout_source(options.subrepo, cache, options.subrepodir, rev=sub_rev)

        with open('SOURCE_CHANGESET', 'w') as fd:
          rev = CheckOutput(['hg', 'parent', '--template={node}'])
          fd.write(rev)

        with open('comm/SOURCE_CHANGESET', 'w') as fd:
          fd.write(sub_rev)

        self.vendor_cbindgen(os.path.join(os.getcwd(), 'third_party/cbindgen'))
        self.vendor_dump_syms(os.path.join(os.getcwd(), 'third_party/dump_syms'))

        l10ndir = 'l10n'

        strings_all_url = ('https://ftp.mozilla.org/pub/%s/candidates/%s-candidates/build%s/source/%s-%s.strings_all.tar.zst'
                           % (basename, version, build, basename, version))
        print('*** Download and extract %s-%s.strings_all.tar.zst ***' % (basename, version))
        with urllib.request.urlopen(strings_all_url) as r:
          unzstd = zstandard.ZstdDecompressor()
          with unzstd.stream_reader(r) as s:
            with tarfile.open(mode="r|", fileobj=s) as tf:
              tf.extractall()
              shutil.move('l10n-central', l10ndir)

        with open(os.path.join(application, 'config/version.txt'), 'r') as vf:
          upstream_version = re.sub(r'~$', '', re.sub(r'([0-9\.]*)(.*)', r'\1~\2', vf.read().strip(), 1))

        if version == None:
          version = upstream_version
          for line in CheckOutput(['hg', 'tip']).split('\n'):
            if line.startswith('changeset:'):
              rev = line.split()[1].split(':')[0].strip()
              changeset = line.split()[1].split(':')[1].strip()
              break

          u = urllib.request.urlopen('%s/pushlog?changeset=%s' % (repo, changeset))
          dom = xml.dom.minidom.parseString(u.read())
          t = time.strptime(dom.getElementsByTagName('updated')[0].firstChild.nodeValue.strip(), '%Y-%m-%dT%H:%M:%SZ')
          version += '~hg%s%s%sr%s' % ('%02d' % t.tm_year, '%02d' % t.tm_mon, '%02d' % t.tm_mday, rev)
          u.close()
        else:
          version = re.sub(r'~$', '', re.sub(r'([0-9\.]*)(.*)', r'\1~\2', version, 1))
          version += '+build%s' % build
          if not version.startswith(upstream_version):
            print("Version '%s' does not match upstream version '%s'" % (version, upstream_version))
            sys.exit(1)

        with open('BUILDID', 'w') as fd:
          fd.write(buildid)

        print('*** Debian package version is %s' % version)
        print('*** Packing tarball ***')
        with ScopedWorkingDirectory('..'):
          topsrcdir = '%s-%s' % (name, version)
          with ScopedRename(name, topsrcdir):
            env = {'XZ_OPT': '--threads=0'}
            args = ['tar', '-Jc', '--exclude=.gitattributes', '--exclude=.hg', '--exclude=.hgignore', '--exclude=.hgtags', '--exclude=.svn']
            for exclude in settings['excludes']:
              args.append('--no-wildcards-match-slash') if exclude['wms'] == False else args.append('--wildcards-match-slash')
              args.append('--exclude')
              args.append(os.path.join(topsrcdir , exclude['path']))
            args.append('-f')
            args.append(os.path.join(orig_cwd, '%s_%s.orig.tar.xz' % (name, version)))
            for include in settings['includes']:
              args.append(os.path.join(topsrcdir, include))

            CheckCall(args, env=env)

def main():
  check_external_dependencies()
  creator = TarballCreator()
  creator.run()

if __name__ == '__main__':
  main()
