This commit is contained in:
2022-05-17 01:56:44 +03:00
commit 3a9b8c8468
117 changed files with 7175 additions and 0 deletions
+393
View File
@@ -0,0 +1,393 @@
package build
import (
"context"
"errors"
"fmt"
"github.com/markbates/pkger"
"io"
"io/ioutil"
"log"
"net"
"net/http"
"net/url"
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"runtime"
"strings"
"time"
)
const (
LatestVersion = "latest"
)
type Requirements struct {
BrowserSource BrowserSource
BrowserChannel string // "beta", "esr", "dev" and so on
DriverVersion string
Tags []string
NoCache bool
TestsDir string
RunTests bool
IgnoreTests bool
PushImage bool
}
type BrowserSource string
// Return regular file corresponding to this source and optionally download this file
func (bs *BrowserSource) Prepare() (string, string, error) {
src := string(*bs)
if src == "" {
return "", "", errors.New("empty browser source")
}
if _, err := os.Stat(src); err == nil {
pkgName := filepath.Base(src)
return src, extractVersion(pkgName), nil
} else if u, err := url.Parse(src); strings.HasPrefix(src, "http") && err == nil {
pkgName := path.Base(src)
data, err := downloadFile(u.String())
if err != nil {
return "", "", fmt.Errorf("download file: %v", err)
}
f, err := ioutil.TempFile("", "images")
if err != nil {
return "", "", fmt.Errorf("temporary file: %v", err)
}
outputFileName := f.Name()
err = ioutil.WriteFile(outputFileName, data, 0644)
if err != nil {
return "", "", fmt.Errorf("save downloaded file: %v", err)
}
return outputFileName, extractVersion(pkgName), nil
}
return "", src, nil
}
func extractVersion(name string) string {
pieces := strings.Split(name, "_")
version := name
if len(pieces) >= 2 {
version = pieces[1]
}
pieces = strings.Split(version, "+")
pieces = strings.Split(pieces[0], "-")
pieces = strings.Split(pieces[0], "~")
return pieces[0]
}
func versionN(pkgVersion string, n int) string {
buildVersion := pkgVersion
pieces := strings.Split(pkgVersion, ".")
if len(pieces) >= n {
buildVersion = strings.Join(pieces[0:n], ".")
}
return buildVersion
}
func majorVersion(pkgVersion string) string {
return versionN(pkgVersion, 1)
}
func majorMinorVersion(pkgVersion string) string {
return versionN(pkgVersion, 2)
}
func buildVersion(pkgVersion string) string {
return versionN(pkgVersion, 3)
}
type Image struct {
Dir string
BuildArgs []string
Labels []string
FileServer bool
Requirements
}
func NewImage(srcDir string, destDir string, req Requirements) (*Image, error) {
if !requireCommand("docker") {
return nil, fmt.Errorf("docker is not installed")
}
dir, err := copySourceFiles(srcDir, destDir)
if err != nil {
return nil, fmt.Errorf("copy source files: %v", err)
}
if len(req.Tags) == 0 {
return nil, errors.New("image tag is required")
}
return &Image{Dir: dir, Requirements: req}, nil
}
func requireCommand(cmd string) bool {
_, err := exec.LookPath(cmd)
return err == nil
}
func tmpDir() (string, error) {
dir, err := ioutil.TempDir("", "images")
if err != nil {
return "", fmt.Errorf("create temporary dir: %v", err)
}
return dir, nil
}
func copySourceFiles(srcDir string, destDir string) (string, error) {
const prefix = "/static"
walkDir := filepath.Join(prefix, srcDir)
err := pkger.Walk(walkDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
regex := regexp.MustCompile(`^.+:/static(.+)$`)
relativePath := regex.FindStringSubmatch(path)[1]
outputPath := filepath.Join(destDir, relativePath)
if info.IsDir() {
return os.MkdirAll(outputPath, info.Mode())
}
fileDir := filepath.Join(destDir, filepath.Dir(relativePath))
if !fileExists(fileDir) {
log.Printf("mkdir dir %s", fileDir)
return os.MkdirAll(fileDir, info.Mode())
}
src, err := pkger.Open(path)
if err != nil {
return err
}
defer src.Close()
dest, err := os.Create(outputPath)
if err != nil {
return err
}
defer dest.Close()
_, err = io.Copy(dest, src)
if err != nil {
return err
}
err = dest.Sync()
if err != nil {
return err
}
err = os.Chmod(outputPath, info.Mode())
if err != nil {
return err
}
return nil
})
if err != nil {
return "", err
}
return filepath.Join(destDir, srcDir), nil
}
func (i *Image) Build() error {
args := []string{"build"}
for _, tag := range i.Tags {
args = append(args, "-t", tag)
}
if len(i.BuildArgs) > 0 {
for _, arg := range i.BuildArgs {
args = append(args, "--build-arg", arg)
}
}
if httpProxy := os.Getenv("HTTP_PROXY"); httpProxy != "" {
args = append(args, "--build-arg", fmt.Sprintf("http_proxy=%s", httpProxy))
}
if httpsProxy := os.Getenv("HTTPS_PROXY"); httpsProxy != "" {
args = append(args, "--build-arg", fmt.Sprintf("https_proxy=%s", httpsProxy))
}
if len(i.Labels) > 0 {
for _, label := range i.Labels {
args = append(args, "--label", label)
}
}
if i.NoCache {
args = append(args, "--no-cache")
}
if i.FileServer {
server := &http.Server{
Handler: http.FileServer(http.Dir(i.Dir)),
}
ln, err := net.Listen("tcp", ":8080")
if err != nil {
return fmt.Errorf("failed to allocate free port: %v", err)
}
e := make(chan error)
go func() {
e <- server.Serve(ln)
}()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
defer server.Shutdown(ctx)
if runtime.GOOS == "linux" {
ip, err := dockerHostIP()
if err != nil {
return fmt.Errorf("failed to detect host machine IP: %v", err)
}
args = append(args, "--add-host", fmt.Sprintf("host.docker.internal:%s", ip))
}
}
args = append(args, i.Dir)
log.Printf("running command: docker %s", strings.Join(args, " "))
cmd := exec.Command("docker", args...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := cmd.Start()
if err != nil {
return fmt.Errorf("failed to start command: %v", err)
}
err = cmd.Wait()
if err != nil {
return fmt.Errorf("command execution error: %v", err)
}
return nil
}
func dockerHostIP() (string, error) {
ifaces, err := net.Interfaces()
if err != nil {
return "", err
}
for _, iface := range ifaces {
if iface.Name != "docker0" {
continue
}
if iface.Flags&net.FlagUp == 0 {
continue // interface down
}
if iface.Flags&net.FlagLoopback != 0 {
continue // loopback interface
}
addrs, err := iface.Addrs()
if err != nil {
return "", err
}
for _, addr := range addrs {
var ip net.IP
switch v := addr.(type) {
case *net.IPNet:
ip = v.IP
case *net.IPAddr:
ip = v.IP
}
if ip == nil || ip.IsLoopback() {
continue
}
ip = ip.To4()
if ip == nil {
continue // not an ipv4 address
}
return ip.String(), nil
}
}
return "", errors.New("no live network interfaces detected")
}
func (i *Image) Test(testsDir string, browserName string, browserVersion string) error {
if !i.RunTests {
log.Println("not running tests")
return nil
}
ref := i.Tags[0]
err := doTest(ref, testsDir, browserName, browserVersion)
if err != nil {
if i.IgnoreTests {
log.Printf("ignoring tests: %v", err)
return nil
}
return fmt.Errorf("tests error: %v", err)
}
log.Println("tests passed")
return nil
}
func doTest(ref string, testsDir string, browserName string, browserVersion string) error {
if !fileExists(testsDir) {
return fmt.Errorf("tests directory %s does not exist", testsDir)
}
seleniumUrl := "http://localhost:4445/"
if browserName == "firefox" || (browserName == "opera" && browserVersion == "12.16") {
seleniumUrl = "http://localhost:4445/wd/hub"
}
exec.Command("docker", "rm", "-f", "selenium").Output()
defer func() {
exec.Command("docker", "rm", "-f", "selenium").Output()
}()
output, err := exec.Command("docker", "run", "-d", "--name", "selenium", "--privileged", "-p", "4445:4444", ref).Output()
if err != nil {
return fmt.Errorf("failed to start docker image: %s %v", string(output), err)
}
if !requireCommand("mvn") {
return fmt.Errorf("maven is not installed")
}
mvnCmd := exec.Command("mvn", "clean", "test",
fmt.Sprintf("-Dgrid.connection.url=%s", seleniumUrl),
fmt.Sprintf("-Dgrid.browser.name=%s", browserName),
fmt.Sprintf("-Dgrid.browser.version=%s", browserVersion),
)
mvnCmd.Dir = testsDir
mvnCmd.Stdout = os.Stdout
mvnCmd.Stderr = os.Stderr
err = mvnCmd.Start()
if err != nil {
return fmt.Errorf("failed to start tests: %v", err)
}
err = mvnCmd.Wait()
if err != nil {
return fmt.Errorf("tests finished with error: %v", err)
}
return nil
}
func fileExists(p string) bool {
_, err := os.Stat(p)
return !os.IsNotExist(err)
}
func (i *Image) Push() error {
if i.PushImage {
for _, tag := range i.Tags {
cmd := exec.Command("docker", "push", tag)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := cmd.Start()
if err != nil {
return fmt.Errorf("invalid docker push %s: %v", tag, err)
}
err = cmd.Wait()
if err != nil {
return fmt.Errorf("pushing failed: %v", err)
}
}
}
return nil
}
+37
View File
@@ -0,0 +1,37 @@
package build
import (
. "github.com/aandryashin/matchers"
"testing"
)
func TestVersionFromPackageName(t *testing.T) {
AssertThat(t, extractVersion("firefox_45.0.2+build1-0ubuntu0.14.04.1+aerokube0_amd64.deb"), EqualTo{"45.0.2"})
AssertThat(t, extractVersion("firefox_71.0_b11+build1-0ubuntu0.18.04.1_amd64.deb"), EqualTo{"71.0"})
AssertThat(t, extractVersion("firefox-trunk_72.0_a1_hg20191114r501767-0ubuntu0.18.04.1_umd1_amd64.deb"), EqualTo{"72.0"})
AssertThat(t, extractVersion("google-chrome-stable_48.0.2564.109-1+aerokube0_amd64.deb"), EqualTo{"48.0.2564.109"})
AssertThat(t, extractVersion("google-chrome-beta_79.0.3945.36-1_amd64.deb"), EqualTo{"79.0.3945.36"})
AssertThat(t, extractVersion("google-chrome-unstable_80.0.3964.0-1_amd64.deb"), EqualTo{"80.0.3964.0"})
AssertThat(t, extractVersion("opera-stable_38.0.2220.31_amd64.deb"), EqualTo{"38.0.2220.31"})
AssertThat(t, extractVersion("opera-beta_65.0.3467.32_amd64.deb"), EqualTo{"65.0.3467.32"})
AssertThat(t, extractVersion("opera-developer_66.0.3502.0_amd64.deb"), EqualTo{"66.0.3502.0"})
AssertThat(t, extractVersion("yandex-browser-beta_19.9.3.358-1_amd64.deb"), EqualTo{"19.9.3.358"})
}
func TestVersionToTag(t *testing.T) {
AssertThat(t, extractVersion("45.0.2+build1-0ubuntu0.14.04.1+aerokube0"), EqualTo{"45.0.2"})
AssertThat(t, extractVersion("48.0.2564.109-1+aerokube0"), EqualTo{"48.0.2564.109"})
AssertThat(t, extractVersion("78.0.3904.108-1"), EqualTo{"78.0.3904.108"})
AssertThat(t, extractVersion("38.0.2220.31"), EqualTo{"38.0.2220.31"})
AssertThat(t, extractVersion("71.0~b11+build1-0ubuntu0.18.04.1"), EqualTo{"71.0"})
AssertThat(t, extractVersion("74.0~a1~hg20200205r512485-0ubuntu0.18.04.1~umd1"), EqualTo{"74.0"})
}
func TestParseVersion(t *testing.T) {
AssertThat(t, majorVersion("78.0.3904.108"), EqualTo{"78"})
AssertThat(t, majorVersion("78"), EqualTo{"78"})
AssertThat(t, majorMinorVersion("78.0.3904.108"), EqualTo{"78.0"})
AssertThat(t, majorMinorVersion("78.0"), EqualTo{"78.0"})
AssertThat(t, buildVersion("78.0.3904.108"), EqualTo{"78.0.3904"})
AssertThat(t, buildVersion("78.0.3904"), EqualTo{"78.0.3904"})
}
+178
View File
@@ -0,0 +1,178 @@
package build
import (
"errors"
"fmt"
"os"
"path/filepath"
"strconv"
)
const (
chromeDriverBinary = "chromedriver"
)
type Chrome struct {
Requirements
}
func (c *Chrome) Build() error {
pkgSrcPath, pkgVersion, err := c.BrowserSource.Prepare()
if err != nil {
return fmt.Errorf("invalid browser source: %v", err)
}
pkgTagVersion := extractVersion(pkgVersion)
driverVersion, err := c.parseChromeDriverVersion(pkgTagVersion)
if err != nil {
return fmt.Errorf("parse chromedriver version: %v", err)
}
// Build dev image
devDestDir, err := tmpDir()
if err != nil {
return fmt.Errorf("create dev temporary dir: %v", err)
}
srcDir := "chrome/apt"
if pkgSrcPath != "" {
srcDir = "chrome/local"
pkgDestDir := filepath.Join(devDestDir, srcDir)
err := os.MkdirAll(pkgDestDir, 0755)
if err != nil {
return fmt.Errorf("create %v temporary dir: %v", pkgDestDir, err)
}
pkgDestPath := filepath.Join(pkgDestDir, "google-chrome.deb")
err = os.Rename(pkgSrcPath, pkgDestPath)
if err != nil {
return fmt.Errorf("move package: %v", err)
}
}
devImageTag := fmt.Sprintf("selenoid/dev_chrome:%s", pkgTagVersion)
devImageRequirements := Requirements{NoCache: c.NoCache, Tags: []string{devImageTag}}
devImage, err := NewImage(srcDir, devDestDir, devImageRequirements)
if err != nil {
return fmt.Errorf("init dev image: %v", err)
}
devBuildArgs := []string{fmt.Sprintf("VERSION=%s", pkgVersion)}
devBuildArgs = append(devBuildArgs, c.channelToBuildArgs()...)
devImage.BuildArgs = devBuildArgs
if pkgSrcPath != "" {
devImage.FileServer = true
}
err = devImage.Build()
if err != nil {
return fmt.Errorf("build dev image: %v", err)
}
// Build main image
destDir, err := tmpDir()
if err != nil {
return fmt.Errorf("create temporary dir: %v", err)
}
image, err := NewImage("chrome", destDir, c.Requirements)
if err != nil {
return fmt.Errorf("init image: %v", err)
}
image.BuildArgs = append(image.BuildArgs, fmt.Sprintf("VERSION=%s", pkgTagVersion))
err = c.downloadChromeDriver(image.Dir, driverVersion)
if err != nil {
return fmt.Errorf("failed to download chromedriver: %v", err)
}
image.Labels = []string{fmt.Sprintf("driver=chromedriver:%s", driverVersion)}
err = image.Build()
if err != nil {
return fmt.Errorf("build image: %v", err)
}
err = image.Test(c.TestsDir, "chrome", pkgTagVersion)
if err != nil {
return fmt.Errorf("test image: %v", err)
}
err = image.Push()
if err != nil {
return fmt.Errorf("push image: %v", err)
}
return nil
}
func (c *Chrome) channelToBuildArgs() []string {
switch c.BrowserChannel {
case "beta":
return []string{"PACKAGE=google-chrome-beta", "INSTALL_DIR=chrome-beta"}
case "dev":
return []string{"PACKAGE=google-chrome-unstable", "INSTALL_DIR=chrome-unstable"}
default:
return []string{}
}
}
func (c *Chrome) parseChromeDriverVersion(pkgVersion string) (string, error) {
version := c.DriverVersion
if version == LatestVersion {
const baseUrl = "https://chromedriver.storage.googleapis.com/"
v, err := c.getLatestChromeDriver(baseUrl, pkgVersion)
if err != nil {
return "", err
}
version = v
}
return version, nil
}
func (c *Chrome) downloadChromeDriver(dir string, version string) error {
u := fmt.Sprintf("http://chromedriver.storage.googleapis.com/%s/chromedriver_linux64.zip", version)
_, err := downloadDriver(u, chromeDriverBinary, dir)
if err != nil {
return fmt.Errorf("download chromedriver: %v", err)
}
return nil
}
func (c *Chrome) getLatestChromeDriver(baseUrl string, pkgVersion string) (string, error) {
fetchVersion := func(url string) (string, error) {
data, err := sendGet(url)
if err != nil {
return "", fmt.Errorf("read chromedriver version: %v", err)
}
return string(data), nil
}
if c.BrowserChannel != "dev" {
chromeBuildVersion := buildVersion(pkgVersion)
u := baseUrl + fmt.Sprintf("LATEST_RELEASE_%s", chromeBuildVersion)
v, err := fetchVersion(u)
if err == nil {
return v, nil
}
}
chromeMajorVersion, err := strconv.Atoi(majorVersion(pkgVersion))
if err != nil {
return "", fmt.Errorf("chrome major version: %v", err)
}
u := baseUrl + fmt.Sprintf("LATEST_RELEASE_%d", chromeMajorVersion)
v, err := fetchVersion(u)
if err == nil {
return v, nil
} else {
previousChromeMajorVersion := chromeMajorVersion - 1
u = baseUrl + fmt.Sprintf("LATEST_RELEASE_%d", previousChromeMajorVersion)
v, err := fetchVersion(u)
if err == nil {
return v, nil
} else {
return "", errors.New("could not find compatible chromedriver")
}
}
}
+122
View File
@@ -0,0 +1,122 @@
package build
import (
"fmt"
"os"
"path/filepath"
)
const (
msedgeDriverBinary = "msedgedriver"
)
type Edge struct {
Requirements
}
func (c *Edge) Build() error {
pkgSrcPath, pkgVersion, err := c.BrowserSource.Prepare()
if err != nil {
return fmt.Errorf("invalid browser source: %v", err)
}
pkgTagVersion := extractVersion(pkgVersion)
// Build dev image
devDestDir, err := tmpDir()
if err != nil {
return fmt.Errorf("create dev temporary dir: %v", err)
}
srcDir := "edge/apt"
if pkgSrcPath != "" {
srcDir = "edge/local"
pkgDestDir := filepath.Join(devDestDir, srcDir)
err := os.MkdirAll(pkgDestDir, 0755)
if err != nil {
return fmt.Errorf("create %v temporary dir: %v", pkgDestDir, err)
}
pkgDestPath := filepath.Join(pkgDestDir, "microsoft-edge.deb")
err = os.Rename(pkgSrcPath, pkgDestPath)
if err != nil {
return fmt.Errorf("move package: %v", err)
}
}
devImageTag := fmt.Sprintf("selenoid/dev_edge:%s", pkgTagVersion)
devImageRequirements := Requirements{NoCache: c.NoCache, Tags: []string{devImageTag}}
devImage, err := NewImage(srcDir, devDestDir, devImageRequirements)
if err != nil {
return fmt.Errorf("init dev image: %v", err)
}
devBuildArgs := []string{fmt.Sprintf("VERSION=%s", pkgVersion)}
devBuildArgs = append(devBuildArgs, c.channelToBuildArgs()...)
devImage.BuildArgs = devBuildArgs
if pkgSrcPath != "" {
devImage.FileServer = true
}
err = devImage.Build()
if err != nil {
return fmt.Errorf("build dev image: %v", err)
}
// Build main image
destDir, err := tmpDir()
if err != nil {
return fmt.Errorf("create temporary dir: %v", err)
}
image, err := NewImage("edge", destDir, c.Requirements)
if err != nil {
return fmt.Errorf("init image: %v", err)
}
image.BuildArgs = append(image.BuildArgs, fmt.Sprintf("VERSION=%s", pkgTagVersion))
driverVersion, err := c.downloadMSEdgeDriver(image.Dir)
if err != nil {
return fmt.Errorf("failed to download msedgedriver: %v", err)
}
image.Labels = []string{fmt.Sprintf("driver=msedgedriver:%s", driverVersion)}
err = image.Build()
if err != nil {
return fmt.Errorf("build image: %v", err)
}
err = image.Test(c.TestsDir, "MicrosoftEdge", pkgTagVersion)
if err != nil {
return fmt.Errorf("test image: %v", err)
}
err = image.Push()
if err != nil {
return fmt.Errorf("push image: %v", err)
}
return nil
}
func (c *Edge) channelToBuildArgs() []string {
switch c.BrowserChannel {
case "beta":
return []string{"PACKAGE=microsoft-edge-beta", "INSTALL_DIR=msedge-beta"}
case "dev":
return []string{"PACKAGE=microsoft-edge-dev", "INSTALL_DIR=msedge-dev"}
default:
return []string{}
}
}
func (c *Edge) downloadMSEdgeDriver(dir string) (string, error) {
version := c.DriverVersion
// Full driver versions list can be fetched as XML from https://msedgedriver.azureedge.net/
u := fmt.Sprintf("https://msedgewebdriverstorage.blob.core.windows.net/edgewebdriver/%s/edgedriver_linux64.zip", version)
_, err := downloadDriver(u, msedgeDriverBinary, dir)
if err != nil {
return "", fmt.Errorf("download msedgedriver: %v", err)
}
return version, nil
}
+326
View File
@@ -0,0 +1,326 @@
package build
import (
"archive/tar"
"archive/zip"
"bufio"
"bytes"
"compress/gzip"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"math/rand"
"net/http"
"os"
"path/filepath"
"regexp"
"strings"
"time"
"gopkg.in/cheggaaa/pb.v1"
)
const (
zipMagicHeader = "504b"
gzipMagicHeader = "1f8b"
)
func downloadFile(url string) ([]byte, error) {
var b bytes.Buffer
w := bufio.NewWriter(&b)
err := downloadFileWithProgressBar(url, w)
if err != nil {
return nil, err
}
w.Flush()
return b.Bytes(), nil
}
func downloadFileWithProgressBar(url string, w io.Writer) error {
resp, err := http.Get(url)
if err != nil {
return fmt.Errorf("file download error: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return fmt.Errorf("unexpected response code: %d", resp.StatusCode)
}
contentLength := int(resp.ContentLength)
writer := w
if contentLength > 0 {
bar := pb.New(contentLength).SetUnits(pb.U_BYTES)
bar.Output = os.Stderr
bar.Start()
defer bar.Finish()
writer = io.MultiWriter(w, bar)
}
_, err = io.Copy(writer, resp.Body)
if err != nil {
return fmt.Errorf("failed to save file: %v", err)
}
return nil
}
func downloadDriver(url string, filename string, outputDir string) (string, error) {
log.Printf("downloading driver from %s", url)
data, err := downloadFile(url)
if err != nil {
return "", fmt.Errorf("failed to download driver archive: %v", err)
}
return extractFile(data, filename, outputDir)
}
func getMagicHeader(data []byte) string {
if len(data) >= 2 {
return hex.EncodeToString(data[:2])
}
return ""
}
func isZipFile(data []byte) bool {
return getMagicHeader(data) == zipMagicHeader
}
func isTarGzFile(data []byte) bool {
return getMagicHeader(data) == gzipMagicHeader
}
func extractFile(data []byte, filename string, outputDir string) (string, error) {
if isZipFile(data) {
return unzip(data, filename, outputDir)
} else if isTarGzFile(data) {
return untar(data, filename, outputDir)
} else {
outputPath := filepath.Join(outputDir, filename)
err := ioutil.WriteFile(outputPath, data, os.ModePerm)
if err != nil {
return "", fmt.Errorf("failed to save file %s: %v", outputPath, err)
}
fmt.Println("Patched chromedriver")
patchDriver(outputPath)
return outputPath, nil
}
}
func patchDriver(filename string) {
rand.Seed(time.Now().UnixNano())
var lettersLower = []byte("abcdefghijklmnopqrstuvwxyz")
var lettersUpper = []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
rand_str_repl := make([]byte, 26)
for i := range rand_str_repl {
if i == 2 {
rand_str_repl[i] = rand_str_repl[0]
} else if i == 3 {
rand_str_repl[i] = byte('_')
} else if i == 20 || i == 21 {
rand_str_repl[i] = lettersUpper[rand.Intn(len(lettersUpper))]
} else {
rand_str_repl[i] = lettersLower[rand.Intn(len(lettersLower))]
}
}
log.Println(string(rand_str_repl))
re := regexp.MustCompile(`cdc_.{22}`)
input_file, err := ioutil.ReadFile(filename)
log.Println(filename)
if err == nil {
match := re.Find(input_file)
if match == nil {
log.Fatalln("cant find driver cdc_")
}
log.Println(string(match))
output_data := re.ReplaceAll(input_file, rand_str_repl)
match = re.Find(output_data)
if match != nil {
log.Fatalln("cant patch driver cdc_")
}
err := ioutil.WriteFile(filename, output_data, os.ModePerm)
if err != nil {
log.Fatalln("cant write to file")
}
}
}
// Based on http://stackoverflow.com/questions/20357223/easy-way-to-unzip-file-with-golang
func unzip(data []byte, fileName string, outputDir string) (string, error) {
zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
// Closure to address file descriptors issue with all the deferred .Close() methods
extractAndWriteFile := func(f *zip.File) (string, error) {
rc, err := f.Open()
if err != nil {
return "", err
}
defer rc.Close()
outputPath := filepath.Join(outputDir, f.Name)
if f.FileInfo().IsDir() {
return "", fmt.Errorf("can only unzip files but %s is a directory", f.Name)
}
err = outputFile(outputPath, f.Mode(), rc)
if err != nil {
return "", err
}
fmt.Println("Patched chromedriver")
patchDriver(outputPath)
return outputPath, nil
}
if err == nil {
for _, f := range zr.File {
if f.Name == fileName {
return extractAndWriteFile(f)
}
}
err = fmt.Errorf("file %s does not exist in archive", fileName)
}
return "", err
}
// Based on https://medium.com/@skdomino/taring-untaring-files-in-go-6b07cf56bc07
func untar(data []byte, fileName string, outputDir string) (string, error) {
gzr, err := gzip.NewReader(bytes.NewReader(data))
defer gzr.Close()
extractAndWriteFile := func(tr *tar.Reader, header *tar.Header) (string, error) {
outputPath := filepath.Join(outputDir, header.Name)
if header.Typeflag == tar.TypeDir {
return "", fmt.Errorf("can only untar files but %s is a directory", header.Name)
}
err = outputFile(outputPath, os.FileMode(header.Mode), tr)
if err != nil {
return "", err
}
return outputPath, nil
}
if err == nil {
tr := tar.NewReader(gzr)
loop:
for {
header, err := tr.Next()
switch {
case err == io.EOF:
break loop
case err != nil:
return "", err
case header == nil:
continue
}
return extractAndWriteFile(tr, header)
}
err = fmt.Errorf("file %s does not exist in archive", fileName)
}
return "", err
}
func outputFile(outputPath string, mode os.FileMode, r io.Reader) error {
os.MkdirAll(filepath.Dir(outputPath), 0755)
f, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
if err != nil {
return err
}
defer f.Close()
_, err = io.Copy(f, r)
if err != nil {
return err
}
return nil
}
func doSendGet(url string, token string) ([]byte, error) {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, fmt.Errorf("invalid request: %v", err)
}
if token != "" {
req.Header.Add("Authorization", fmt.Sprintf("token %s", token))
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, fmt.Errorf("request error: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unsuccessful response: %d %s", resp.StatusCode, resp.Status)
}
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("read response body: %v", err)
}
return data, nil
}
func sendGet(url string) ([]byte, error) {
return doSendGet(url, "")
}
func sendGetWithAuth(url string, token string) ([]byte, error) {
return doSendGet(url, token)
}
func latestGithubRelease(repo string) (string, error) {
token := os.Getenv("GITHUB_TOKEN")
url := fmt.Sprintf("https://api.github.com/repos/%s/releases/latest", repo)
data, err := sendGetWithAuth(url, token)
if err != nil {
return "", fmt.Errorf("get latest github release data: %v", err)
}
type info struct {
TagName string `json:"tag_name"`
}
var i info
err = json.Unmarshal(data, &i)
if err != nil {
return "", fmt.Errorf("json unmarshal: %v", err)
}
return i.TagName, nil
}
func latestGithubLinuxAsset(repo string) (string, error) {
token := os.Getenv("GITHUB_TOKEN")
url := fmt.Sprintf("https://api.github.com/repos/%s/releases", repo)
data, err := sendGetWithAuth(url, token)
if err != nil {
return "", fmt.Errorf("get github releases data: %v", err)
}
type AssetInfo struct {
Name string `json:"name"`
}
type Release struct {
Assets []AssetInfo `json:"assets"`
}
type Releases []Release
var releases Releases
err = json.Unmarshal(data, &releases)
if err != nil {
return "", fmt.Errorf("json unmarshal: %v", err)
}
for _, release := range releases {
for _, asset := range release.Assets {
if strings.Contains(asset.Name, "linux") {
return asset.Name, nil
}
}
}
return "", fmt.Errorf("could not find github linux asset")
}
+65
View File
@@ -0,0 +1,65 @@
package build
import (
. "github.com/aandryashin/matchers"
"io/ioutil"
"os"
"testing"
)
func TestUnzip(t *testing.T) {
data := readFile(t, "testfile.zip")
AssertThat(t, isZipFile(data), Is{true})
AssertThat(t, isTarGzFile(data), Is{false})
testUnpack(t, data, "zip-testfile", func(data []byte, filePath string, outputDir string) (string, error) {
return unzip(data, filePath, outputDir)
}, "zip\n")
}
func TestUntar(t *testing.T) {
data := readFile(t, "testfile.tar.gz")
AssertThat(t, isTarGzFile(data), Is{true})
AssertThat(t, isZipFile(data), Is{false})
testUnpack(t, data, "gzip-testfile", func(data []byte, filePath string, outputDir string) (string, error) {
return untar(data, filePath, outputDir)
}, "gzip\n")
}
func withTmpDir(t *testing.T, prefix string, fn func(*testing.T, string)) {
dir, err := ioutil.TempDir("", prefix)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
fn(t, dir)
}
func testUnpack(t *testing.T, data []byte, fileName string, fn func([]byte, string, string) (string, error), correctContents string) {
withTmpDir(t, "test-unpack", func(t *testing.T, dir string) {
unpackedFile, err := fn(data, fileName, dir)
if err != nil {
t.Fatal(err)
}
if !fileExists(unpackedFile) {
t.Fatalf("file %s does not exist\n", unpackedFile)
}
unpackedFileContents := string(readFile(t, unpackedFile))
if unpackedFileContents != correctContents {
t.Fatalf("incorrect unpacked file contents; expected: '%s', actual: '%s'\n", correctContents, unpackedFileContents)
}
})
}
func readFile(t *testing.T, fileName string) []byte {
data, err := ioutil.ReadFile(fileName)
if err != nil {
t.Fatal(err)
}
return data
}
+228
View File
@@ -0,0 +1,228 @@
package build
import (
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"strings"
)
const (
geckoDriverBinary = "geckodriver"
)
type Firefox struct {
SelenoidVersion string
SeleniumVersion string
Requirements
}
func (c *Firefox) Build() error {
if c.SelenoidVersion == "" && c.SeleniumVersion == "" {
return errors.New("missing Selenoid or Selenium JAR version")
}
// Build dev image
devDestDir, err := tmpDir()
if err != nil {
return fmt.Errorf("create dev temporary dir: %v", err)
}
devSrcDir := "firefox/apt"
pkgSrcPath, pkgVersion, err := c.BrowserSource.Prepare()
if err != nil {
return fmt.Errorf("invalid browser source: %v", err)
}
if pkgSrcPath != "" {
devSrcDir = "firefox/local"
pkgDestDir := filepath.Join(devDestDir, devSrcDir)
err := os.MkdirAll(pkgDestDir, 0755)
if err != nil {
return fmt.Errorf("create %v temporary dir: %v", pkgDestDir, err)
}
pkgDestPath := filepath.Join(pkgDestDir, "firefox.deb")
err = os.Rename(pkgSrcPath, pkgDestPath)
if err != nil {
return fmt.Errorf("move package: %v", err)
}
}
pkgTagVersion := extractVersion(pkgVersion)
devImageTag := fmt.Sprintf("selenoid/dev_firefox:%s", pkgTagVersion)
devImageRequirements := Requirements{NoCache: c.NoCache, Tags: []string{devImageTag}}
devImage, err := NewImage(devSrcDir, devDestDir, devImageRequirements)
if err != nil {
return fmt.Errorf("init dev image: %v", err)
}
devBuildArgs := []string{fmt.Sprintf("VERSION=%s", pkgVersion)}
devBuildArgs = append(devBuildArgs, c.channelToBuildArgs()...)
devImage.BuildArgs = devBuildArgs
if pkgSrcPath != "" {
devImage.FileServer = true
}
err = devImage.Build()
if err != nil {
return fmt.Errorf("build dev image: %v", err)
}
// Build main image
destDir, err := tmpDir()
if err != nil {
return fmt.Errorf("create temporary dir: %v", err)
}
firefoxMajorVersion, err := strconv.Atoi(majorVersion(pkgTagVersion))
geckoDriverCompatible := firefoxMajorVersion > 48
srcDir := "firefox/selenoid"
if !geckoDriverCompatible {
srcDir = "firefox/selenium"
}
image, err := NewImage(srcDir, destDir, c.Requirements)
if err != nil {
return fmt.Errorf("init dev image: %v", err)
}
image.BuildArgs = append(image.BuildArgs, fmt.Sprintf("VERSION=%s", pkgTagVersion))
firefoxMajorMinorVersion := majorMinorVersion(pkgTagVersion)
if geckoDriverCompatible {
driverVersion, err := c.downloadGeckoDriver(image.Dir)
if err != nil {
return fmt.Errorf("failed to download geckodriver: %v", err)
}
labels := []string{fmt.Sprintf("driver=geckodriver:%s", driverVersion)}
selenoidVersion, err := c.downloadSelenoid(image.Dir)
if err != nil {
return fmt.Errorf("failed to download Selenoid: %v", err)
}
labels = append(labels, fmt.Sprintf("selenoid=%s", selenoidVersion))
image.Labels = labels
browsersJsonFile := filepath.Join(image.Dir, "browsers.json")
data, err := ioutil.ReadFile(browsersJsonFile)
if err != nil {
return fmt.Errorf("failed to read browsers.json: %v", err)
}
newContents := strings.Replace(string(data), "@@VERSION@@", firefoxMajorMinorVersion, -1)
err = ioutil.WriteFile(browsersJsonFile, []byte(newContents), 0)
if err != nil {
return fmt.Errorf("failed to update browsers.json: %v", err)
}
} else {
driverVersion, err := c.downloadSeleniumJAR(image.Dir)
if err != nil {
return fmt.Errorf("failed to download Selenium JAR: %v", err)
}
image.Labels = []string{fmt.Sprintf("driver=selenium:%s", driverVersion)}
}
err = image.Build()
if err != nil {
return fmt.Errorf("build image: %v", err)
}
err = image.Test(c.TestsDir, "firefox", firefoxMajorMinorVersion)
if err != nil {
return fmt.Errorf("test image: %v", err)
}
err = image.Push()
if err != nil {
return fmt.Errorf("push image: %v", err)
}
return nil
}
func (c *Firefox) channelToBuildArgs() []string {
switch c.BrowserChannel {
case "beta":
return []string{"PPA=ppa:mozillateam/firefox-next"}
case "dev":
return []string{"PACKAGE=firefox-trunk", "PPA=ppa:ubuntu-mozilla-daily/ppa"}
case "esr":
return []string{"PACKAGE=firefox-esr", "PPA=ppa:mozillateam/ppa"}
default:
return []string{}
}
}
func (c *Firefox) downloadGeckoDriver(dir string) (string, error) {
version := c.DriverVersion
if version == LatestVersion {
v, err := latestGithubRelease("mozilla/geckodriver")
if err != nil {
return "", fmt.Errorf("latest geckodriver version: %v", err)
}
version = strings.TrimPrefix(v, "v")
}
u := fmt.Sprintf("https://github.com/mozilla/geckodriver/releases/download/v%s/geckodriver-v%s-linux64.tar.gz", version, version)
_, err := downloadDriver(u, geckoDriverBinary, dir)
if err != nil {
return "", fmt.Errorf("download geckodriver: %v", err)
}
return version, nil
}
func (c *Firefox) downloadSelenoid(dir string) (string, error) {
version := c.SelenoidVersion
if version == LatestVersion {
v, err := latestGithubRelease("aerokube/selenoid")
if err != nil {
return "", fmt.Errorf("latest Selenoid version: %v", err)
}
version = v
}
u := fmt.Sprintf("https://github.com/aerokube/selenoid/releases/download/%s/selenoid_linux_amd64", version)
data, err := downloadFile(u)
if err != nil {
return "", fmt.Errorf("download Selenoid: %v", err)
}
outputPath := filepath.Join(dir, "selenoid")
err = ioutil.WriteFile(outputPath, data, 0755)
if err != nil {
return "", fmt.Errorf("save Selenoid: %v", err)
}
return version, nil
}
func (c *Firefox) downloadSeleniumJAR(dir string) (string, error) {
version := c.SeleniumVersion
var u string
switch version {
case "2.15.0", "2.19.0", "2.20.0", "2.21.0", "2.25.0", "2.32.0", "2.35.0", "2.37.0", "2.39.0", "2.40.0", "2.41.0", "2.43.1", "2.44.0", "2.45.0", "2.48.2":
u = fmt.Sprintf("https://repo.jenkins-ci.org/releases/org/seleniumhq/selenium/selenium-server-standalone/%s/selenium-server-standalone-$selenium_version.jar", version)
case "2.47.1":
u = "http://selenium-release.storage.googleapis.com/2.47/selenium-server-standalone-2.47.1.jar"
case "2.53.1":
u = "http://selenium-release.storage.googleapis.com/2.53/selenium-server-standalone-2.53.1.jar"
case "3.2.0":
u = "http://selenium-release.storage.googleapis.com/3.2/selenium-server-standalone-3.2.0.jar"
case "3.3.1":
u = "http://selenium-release.storage.googleapis.com/3.3/selenium-server-standalone-3.3.1.jar"
case "3.4.0":
u = "https://selenium-release.storage.googleapis.com/3.4/selenium-server-standalone-3.4.0.jar"
default:
return "", fmt.Errorf("unsupported Selenium JAR version: %s", version)
}
data, err := downloadFile(u)
if err != nil {
return "", fmt.Errorf("download Selenium JAR: %v", err)
}
outputPath := filepath.Join(dir, "selenium-server-standalone.jar")
err = ioutil.WriteFile(outputPath, data, 0644)
if err != nil {
return "", fmt.Errorf("save Selenium JAR: %v", err)
}
return version, nil
}
+128
View File
@@ -0,0 +1,128 @@
package build
import (
"fmt"
"os"
"path/filepath"
"strings"
)
const (
operaDriverBinary = "operadriver_linux64/operadriver"
)
type Opera struct {
Requirements
}
func (o *Opera) Build() error {
// Build dev image
devDestDir, err := tmpDir()
if err != nil {
return fmt.Errorf("create dev temporary dir: %v", err)
}
srcDir := "opera/apt"
pkgSrcPath, pkgVersion, err := o.BrowserSource.Prepare()
if err != nil {
return fmt.Errorf("invalid browser source: %v", err)
}
if pkgSrcPath != "" {
srcDir = "opera/local"
pkgDestDir := filepath.Join(devDestDir, srcDir)
err := os.MkdirAll(pkgDestDir, 0755)
if err != nil {
return fmt.Errorf("create %v temporary dir: %v", pkgDestDir, err)
}
pkgDestPath := filepath.Join(pkgDestDir, "opera.deb")
err = os.Rename(pkgSrcPath, pkgDestPath)
if err != nil {
return fmt.Errorf("move package: %v", err)
}
}
pkgTagVersion := extractVersion(pkgVersion)
devImageTag := fmt.Sprintf("selenoid/dev_opera:%s", pkgTagVersion)
devImageRequirements := Requirements{NoCache: o.NoCache, Tags: []string{devImageTag}}
devImage, err := NewImage(srcDir, devDestDir, devImageRequirements)
if err != nil {
return fmt.Errorf("init dev image: %v", err)
}
devBuildArgs := []string{fmt.Sprintf("VERSION=%s", pkgVersion)}
devBuildArgs = append(devBuildArgs, o.channelToBuildArgs()...)
devImage.BuildArgs = devBuildArgs
if pkgSrcPath != "" {
devImage.FileServer = true
}
err = devImage.Build()
if err != nil {
return fmt.Errorf("build dev image: %v", err)
}
// Build main image
destDir, err := tmpDir()
if err != nil {
return fmt.Errorf("create temporary dir: %v", err)
}
image, err := NewImage("opera", destDir, o.Requirements)
if err != nil {
return fmt.Errorf("init image: %v", err)
}
image.BuildArgs = append(image.BuildArgs, fmt.Sprintf("VERSION=%s", pkgTagVersion))
driverVersion, err := o.downloadOperaDriver(image.Dir)
if err != nil {
return fmt.Errorf("failed to download operadriver: %v", err)
}
image.Labels = []string{fmt.Sprintf("driver=operadriver:%s", driverVersion)}
err = image.Build()
if err != nil {
return fmt.Errorf("build image: %v", err)
}
err = image.Test(o.TestsDir, "opera", pkgTagVersion)
if err != nil {
return fmt.Errorf("test image: %v", err)
}
err = image.Push()
if err != nil {
return fmt.Errorf("push image: %v", err)
}
return nil
}
func (o *Opera) channelToBuildArgs() []string {
switch o.BrowserChannel {
case "beta":
return []string{"PACKAGE=opera-beta"}
case "dev":
return []string{"PACKAGE=opera-developer"}
default:
return []string{}
}
}
func (o *Opera) downloadOperaDriver(dir string) (string, error) {
version := o.DriverVersion
if version == LatestVersion {
v, err := latestGithubRelease("operasoftware/operachromiumdriver")
if err != nil {
return "", fmt.Errorf("latest Operadriver version: %v", err)
}
version = strings.TrimPrefix(v, "v.")
}
u := fmt.Sprintf("https://github.com/operasoftware/operachromiumdriver/releases/download/v.%s/operadriver_linux64.zip", version)
_, err := downloadDriver(u, operaDriverBinary, dir)
if err != nil {
return "", fmt.Errorf("download Operadriver: %v", err)
}
return version, nil
}
Binary file not shown.
Binary file not shown.
+117
View File
@@ -0,0 +1,117 @@
package build
import (
"fmt"
"os"
"path/filepath"
"strings"
)
const (
yandexDriverBinary = "yandexdriver"
)
type YandexBrowser struct {
Requirements
}
func (yb *YandexBrowser) Build() error {
// Build dev image
devDestDir, err := tmpDir()
if err != nil {
return fmt.Errorf("create dev temporary dir: %v", err)
}
srcDir := "yandex/apt"
pkgSrcPath, pkgVersion, err := yb.BrowserSource.Prepare()
if err != nil {
return fmt.Errorf("invalid browser source: %v", err)
}
if pkgSrcPath != "" {
srcDir = "yandex/local"
pkgDestDir := filepath.Join(devDestDir, srcDir)
err := os.MkdirAll(pkgDestDir, 0755)
if err != nil {
return fmt.Errorf("create %v temporary dir: %v", pkgDestDir, err)
}
pkgDestPath := filepath.Join(pkgDestDir, "yandex-browser.deb")
err = os.Rename(pkgSrcPath, pkgDestPath)
if err != nil {
return fmt.Errorf("move package: %v", err)
}
}
pkgTagVersion := extractVersion(pkgVersion)
devImageTag := fmt.Sprintf("selenoid/dev_yandex:%s", pkgTagVersion)
devImageRequirements := Requirements{NoCache: yb.NoCache, Tags: []string{devImageTag}}
devImage, err := NewImage(srcDir, devDestDir, devImageRequirements)
if err != nil {
return fmt.Errorf("init dev image: %v", err)
}
devBuildArgs := []string{fmt.Sprintf("VERSION=%s", pkgVersion)}
devImage.BuildArgs = devBuildArgs
if pkgSrcPath != "" {
devImage.FileServer = true
}
err = devImage.Build()
if err != nil {
return fmt.Errorf("build dev image: %v", err)
}
// Build main image
destDir, err := tmpDir()
if err != nil {
return fmt.Errorf("create temporary dir: %v", err)
}
image, err := NewImage("yandex", destDir, yb.Requirements)
if err != nil {
return fmt.Errorf("init image: %v", err)
}
image.BuildArgs = append(image.BuildArgs, fmt.Sprintf("VERSION=%s", pkgTagVersion))
driverVersion, err := yb.downloadYandexDriver(image.Dir)
if err != nil {
return fmt.Errorf("failed to download yandexdriver: %v", err)
}
image.Labels = []string{fmt.Sprintf("driver=yandexdriver:%s", driverVersion)}
err = image.Build()
if err != nil {
return fmt.Errorf("build image: %v", err)
}
err = image.Test(yb.TestsDir, "yandex", pkgTagVersion)
if err != nil {
return fmt.Errorf("test image: %v", err)
}
err = image.Push()
if err != nil {
return fmt.Errorf("push image: %v", err)
}
return nil
}
func (yb *YandexBrowser) downloadYandexDriver(dir string) (string, error) {
version := yb.DriverVersion
if version == LatestVersion {
a, err := latestGithubLinuxAsset("yandex/YandexDriver")
if err != nil {
return "", fmt.Errorf("latest yandexdriver linux asset: %v", err)
}
version = strings.Split(a, "-")[1]
}
bv := buildVersion(version)
u := fmt.Sprintf("https://github.com/yandex/YandexDriver/releases/download/v%s-stable/yandexdriver-%s-linux.zip", bv, version)
_, err := downloadDriver(u, yandexDriverBinary, dir)
if err != nil {
return "", fmt.Errorf("download Yandexdriver: %v", err)
}
return version, nil
}