#!/usr/bin/env python3 from urllib.request import urlopen from urllib.error import HTTPError import json from datetime import datetime, date from collections import OrderedDict import subprocess import re import os from pathlib import Path import filecmp with open("current-linux-os-releases.json", "r") as fin: oldData = json.load(fin) with open("projects.json", "r") as fin: projects = json.load(fin) print(oldData) ubuntu = json.loads(urlopen("https://endoflife.date/api/ubuntu.json").read()) debian = json.loads(urlopen("https://endoflife.date/api/debian.json").read()) fedora = json.loads(urlopen("https://endoflife.date/api/fedora.json").read()) centos = json.loads(urlopen("https://endoflife.date/api/centos.json").read()) almalinux = json.loads(urlopen("https://endoflife.date/api/almalinux.json").read()) today = date.today() debLts = set() debAll = set() rpmLts = set() rpmAll = set() dockerImages = {} eol = {} manualIgnore = set(["bionic"]) def debName(data, dist, shortname, endDate, lts, all, dockerImages): for e in sorted(data, key=lambda e: e["releaseDate"]): edate = datetime.strptime(e[endDate], '%Y-%m-%d').date() if 'stream' in e['cycle']: continue if (edate < today): continue # ignore codename on Fedora! if dist != 'fedora' and 'codename' in e: codename = e['codename'].lower().split(" ")[0] if codename in manualIgnore: continue releasename = codename else: codename = shortname + e['cycle'] releasename = e['cycle'] if e['lts'] or ("debian" in e.get('link','')): lts.add(codename) else: print("Not LTS: ", codename) all.add(codename) eol[codename] = e[endDate] dockerImages[codename] = "%s:%s" % (dist,releasename) debName(ubuntu, 'ubuntu', '', 'eol', debLts, debAll, dockerImages) debName(debian, 'debian', '', 'eol', debLts, debAll, dockerImages) debName(centos, 'centos', 'el', 'eol', rpmLts, rpmAll, dockerImages) debName(almalinux, 'almalinux', 'el', 'eol', rpmLts, rpmAll, dockerImages) debName(fedora, 'fedora', 'fc', 'eol', rpmLts, rpmAll, dockerImages) debOld = debLts.intersection(set(['bionic','buster'])) debArchOS = {"i386": sorted(debOld), "amd64": sorted(debAll), "armhf": sorted(debLts), "arm64": sorted(debLts)} for codename in debAll: if codename in oldData["deb-all"]: continue for target in ["nightly", "stable", "release"]: for arch in ["amd64", "arm64", "armhf"]: if codename not in debLts and arch != "amd64": continue dockerImage = "docker.openmodelica.org/build-deps:%s.%s.%s" % (codename,target,arch) nightlyDockerImage = "docker.openmodelica.org/build-deps:%s.nightly.%s" % (codename,arch) path = Path('build/%s.%s.%s' % (codename,target,arch)) nightlyPath = Path('build/%s.nightly.%s' % (codename,arch)) path.mkdir(parents=True, exist_ok=True) subprocess.check_output(["wget", "-q", "https://build.openmodelica.org/apt/pool/contrib/openmodelica_%s-1.dsc" % projects["Projects"]["OpenModelica"][target], "-O", str(path) + "/openmodelica.dsc"]) subprocess.check_output(["sh", "-c", "grep ^Build-Depends "+str(path)+"/openmodelica.dsc > "+str(path)+"/openmodelica-build-deps"]) try: subprocess.check_output(["docker", "pull", dockerImage]) except: subprocess.check_output(["cp", "libiomp-fake", str(path)]) subprocess.check_output(["cp", "Dockerfile.deb.template", str(path)+"/Dockerfile"]) if target != "nightly" and filecmp.cmp(str(path)+"/openmodelica-build-deps", str(nightlyPath)+"/openmodelica-build-deps"): subprocess.check_output(["docker", "tag", nightlyDockerImage, dockerImage]) else: subprocess.check_output(["docker", "buildx", "build", "--platform=linux/%s" % arch, "--build-arg", "REPO=%s" % dockerImages[codename], "-t", dockerImage, str(path)]) subprocess.check_output(["docker", "push", dockerImage]) for codename in rpmAll: if codename in oldData["rpm"]: continue dockerImage = "docker.openmodelica.org/build-deps:%s.amd64" % codename path = Path('build/%s' % codename) path.mkdir(parents=True, exist_ok=True) try: print("Trying to pull: %s" % dockerImage) subprocess.check_output(["docker", "pull", dockerImage]) except: print("Failed to pull: %s, trying to build it" % dockerImage) try: subprocess.check_output(["wget", "-q", "https://raw.githubusercontent.com/OpenModelica/OpenModelicaBuildScripts/master/rpm/SPECS/openmodelica.spec.tpl", "-O", str(path) + "/openmodelica.spec.tpl"]) subprocess.check_output(["cp", "Dockerfile.rpm-%s.template" % codename, str(path)+"/Dockerfile"]) subprocess.check_output(["docker", "buildx", "build", "--platform=linux/amd64", "--build-arg", "REPO=%s" % dockerImages[codename], "-t", dockerImage, str(path)]) subprocess.check_output(["docker", "push", dockerImage]) except: print("Failed to build image: %s" % dockerImage) pkgVersions = {"cmake": {}, "gcc": {}, "clang": {}} pkgsRpmCommand = {"cmake": ["cmake3 --version", "cmake --version"], "gcc": ["/opt/rh/devtoolset-*/root/usr/bin/gcc --version", "gcc --version"], "clang": ["clang --version || '-'"]} for pkg in pkgVersions.keys(): for osrelease in ["qa","ubuntu"]: for line in subprocess.check_output(["rmadison", "-u", osrelease, pkg]).decode("utf-8").split("\n"): e = line.split("|") if len(e) == 1: continue version = e[1].strip().split(":")[-1].split("-")[0].split("~")[0].split("+")[0] codename = e[2].strip().split("/")[0] if codename not in debAll: continue pkgVersions[pkg][codename] = version for osrelease in sorted(rpmAll): version = None subprocess.check_output(["docker", "pull", "docker.openmodelica.org/build-deps:%s.amd64" % osrelease]) for cmd in pkgsRpmCommand[pkg]: command = ["docker", "run", "--rm", "docker.openmodelica.org/build-deps:%s.amd64" % osrelease, "sh", "-c", "%s" % cmd] try: version = subprocess.check_output(command, stderr=subprocess.DEVNULL).decode("utf-8").strip().split("\n")[0].strip() version = re.search('\d+[.]\d+[.]\d+', version).group() # print("Version %s %s: %s" % (os,pkg,version)) break except Exception as e: # print("Failed: " + " ".join(command)) # print(e) pass if version is None: version = "-" pkgVersions[pkg][osrelease] = version if not rpmAll: raise Exception("Found 0 RPM releases") if not debLts: raise Exception("Found 0 deb LTS releases") if not debAll: raise Exception("Found 0 deb releases") with open("current-linux-os-releases.json", "w") as fout: fout.write(json.dumps({ "deb-lts": sorted(debLts), "deb-all": sorted(debAll), "deb-old": sorted(debOld), "deb-arch-os": debArchOS, "rpm": sorted(rpmAll), "docker": dockerImages, "eol": eol, "pkg": pkgVersions },sort_keys=True, indent=True)) print("Created current-linux-os-releases.json. Now checking repos.") subprocess.check_output(["mkdir", "-p", "-m", "775"] + ["/var/www/build.openmodelica.org/apt/pool/contrib-%s" % osrelease for osrelease in debAll]) subprocess.check_output(["mkdir", "-p", "-m", "775"] + ["/var/www/build.openmodelica.org/rpm/%s" % osrelease for osrelease in rpmAll]) for osrelease in rpmAll: fname = "/var/www/build.openmodelica.org/rpm/%s/omc.repo" % osrelease if not os.path.exists(fname): with open(fname, "w") as fout: fout.write("""[openmodelica] name=OpenModelica RPM Repository baseurl=https://build.openmodelica.org/rpm/%s/ enabled=1 gpgcheck=0 """ % osrelease) for arch in debArchOS.keys(): for os in debArchOS[arch]: for release in ["nightly","stable","release"]: try: subprocess.check_output(["docker", "manifest", "inspect", "docker.openmodelica.org/build-deps:%s.%s.%s" % (os,release,arch)]) except: print("Failed to run: docker manifest inspect docker.openmodelica.org/build-deps:%s.%s.%s! Skipping ... " % (os,release,arch)) # We checked this with docker run above # for os in rpmAll: # for arch in ["amd64"]: # subprocess.check_output(["docker", "manifest", "inspect", "docker.openmodelica.org/build-deps:%s.%s" % (os,arch)])