diff --git a/actions.go b/actions.go index 9cd41c1..75fe56c 100644 --- a/actions.go +++ b/actions.go @@ -8,12 +8,11 @@ import ( "os" "time" + "github.com/Davincible/chromedp-undetected/util/easyjson" "github.com/chromedp/cdproto/cdp" "github.com/chromedp/cdproto/emulation" "github.com/chromedp/cdproto/network" "github.com/chromedp/chromedp" - - "github.com/Davincible/chromedp-undetected/util/easyjson" ) // Cookie is used to set browser cookies. diff --git a/chrome_unix.go b/chrome_unix.go new file mode 100644 index 0000000..70e19bc --- /dev/null +++ b/chrome_unix.go @@ -0,0 +1,177 @@ +//go:build unix + +// Package chromedpundetected provides a chromedp context with an undetected +// Chrome browser. +package chromedpundetected + +import ( + "context" + "net" + "os" + "os/exec" + "path" + "strconv" + "strings" + "syscall" + + "github.com/Xuanwo/go-locale" + "github.com/chromedp/chromedp" + "github.com/google/uuid" + "golang.org/x/exp/slog" +) + +var ( + // DefaultUserDirPrefix Defaults. + DefaultUserDirPrefix = "chromedp-undetected-" +) + +// New creates a context with an undetected Chrome executor. +func New(config Config) (context.Context, context.CancelFunc, error) { + var opts []chromedp.ExecAllocatorOption + + userDataDir := path.Join(os.TempDir(), DefaultUserDirPrefix+uuid.NewString()) + if len(config.ChromePath) > 0 { + userDataDir = config.ChromePath + } + + headlessOpts, closeFrameBuffer, err := headlessFlag(config) + if err != nil { + return nil, func() {}, err + } + + opts = append(opts, localeFlag()) + opts = append(opts, supressWelcomeFlag()...) + opts = append(opts, logLevelFlag(config)) + opts = append(opts, debuggerAddrFlag(config)...) + opts = append(opts, noSandboxFlag(config)...) + opts = append(opts, chromedp.UserDataDir(userDataDir)) + opts = append(opts, headlessOpts...) + opts = append(opts, config.ChromeFlags...) + + ctx := context.Background() + if config.Ctx != nil { + ctx = config.Ctx + } + + cancelT := func() {} + if config.Timeout > 0 { + ctx, cancelT = context.WithTimeout(ctx, config.Timeout) + } + + ctx, cancelA := chromedp.NewExecAllocator(ctx, opts...) + ctx, cancelC := chromedp.NewContext(ctx, config.ContextOptions...) + + cancel := func() { + cancelT() + cancelA() + cancelC() + + if err := closeFrameBuffer(); err != nil { + slog.Error("failed to close Xvfb", err) + } + + if len(config.ChromePath) == 0 { + _ = os.RemoveAll(userDataDir) //nolint:errcheck + } + } + + return ctx, cancel, nil +} + +func supressWelcomeFlag() []chromedp.ExecAllocatorOption { + return []chromedp.ExecAllocatorOption{ + chromedp.Flag("no-first-run", true), + chromedp.Flag("no-default-browser-check", true), + } +} + +func debuggerAddrFlag(config Config) []chromedp.ExecAllocatorOption { + port := strconv.Itoa(config.Port) + if config.Port == 0 { + port = getRandomPort() + } + + return []chromedp.ExecAllocatorOption{ + chromedp.Flag("remote-debugging-host", "127.0.0.1"), + chromedp.Flag("remote-debugging-port", port), + } +} + +func localeFlag() chromedp.ExecAllocatorOption { + lang := "en-US" + if tag, err := locale.Detect(); err != nil && len(tag.String()) > 0 { + lang = tag.String() + } + + return chromedp.Flag("lang", lang) +} + +func noSandboxFlag(config Config) []chromedp.ExecAllocatorOption { + var opts []chromedp.ExecAllocatorOption + + if config.NoSandbox { + opts = append(opts, + chromedp.Flag("no-sandbox", true), + chromedp.Flag("test-type", true)) + } + + return opts +} + +func logLevelFlag(config Config) chromedp.ExecAllocatorOption { + return chromedp.Flag("log-level", strconv.Itoa(config.LogLevel)) +} + +func headlessFlag(config Config) ([]chromedp.ExecAllocatorOption, func() error, error) { + var opts []chromedp.ExecAllocatorOption + + cleanup := func() error { return nil } + + if config.Headless { + // Create virtual display + frameBuffer, err := newFrameBuffer("1920x1080x24") + if err != nil { + return nil, nil, err + } + + cleanup = frameBuffer.Stop + + opts = append(opts, + // chromedp.Flag("headless", true), + chromedp.Flag("window-size", "1920,1080"), + chromedp.Flag("start-maximized", true), + chromedp.Flag("no-sandbox", true), + chromedp.ModifyCmdFunc(func(cmd *exec.Cmd) { + cmd.Env = append(cmd.Env, "DISPLAY=:"+frameBuffer.Display) + cmd.Env = append(cmd.Env, "XAUTHORITY="+frameBuffer.AuthPath) + + // Default modify command per chromedp + if _, ok := os.LookupEnv("LAMBDA_TASK_ROOT"); ok { + // do nothing on AWS Lambda + return + } + + if cmd.SysProcAttr == nil { + cmd.SysProcAttr = new(syscall.SysProcAttr) + } + + // When the parent process dies (Go), kill the child as well. + cmd.SysProcAttr.Pdeathsig = syscall.SIGKILL + }), + ) + } + + return opts, cleanup, nil +} + +func getRandomPort() string { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err == nil { + addr := l.Addr().String() + _ = l.Close() //nolint:errcheck,gosec + + return strings.Split(addr, ":")[1] + } + + return "42069" +} diff --git a/chrome_windows.go b/chrome_windows.go new file mode 100644 index 0000000..f660dd1 --- /dev/null +++ b/chrome_windows.go @@ -0,0 +1,120 @@ +// Package chromedpundetected provides a chromedp context with an undetected +// Chrome browser. +package chromedpundetected + +import ( + "context" + "net" + "os" + "path" + "strconv" + "strings" + + "github.com/Xuanwo/go-locale" + "github.com/chromedp/chromedp" + "github.com/google/uuid" +) + +var ( + // DefaultUserDirPrefix Defaults. + DefaultUserDirPrefix = "chromedp-undetected-" +) + +// New creates a context with an undetected Chrome executor. +func New(config Config) (context.Context, context.CancelFunc, error) { + var opts []chromedp.ExecAllocatorOption + + userDataDir := path.Join(os.TempDir(), DefaultUserDirPrefix+uuid.NewString()) + if len(config.ChromePath) > 0 { + userDataDir = config.ChromePath + } + + opts = append(opts, localeFlag()) + opts = append(opts, supressWelcomeFlag()...) + opts = append(opts, logLevelFlag(config)) + opts = append(opts, debuggerAddrFlag(config)...) + opts = append(opts, noSandboxFlag(config)...) + opts = append(opts, chromedp.UserDataDir(userDataDir)) + opts = append(opts, config.ChromeFlags...) + + ctx := context.Background() + if config.Ctx != nil { + ctx = config.Ctx + } + + cancelT := func() {} + if config.Timeout > 0 { + ctx, cancelT = context.WithTimeout(ctx, config.Timeout) + } + + ctx, cancelA := chromedp.NewExecAllocator(ctx, opts...) + ctx, cancelC := chromedp.NewContext(ctx, config.ContextOptions...) + + cancel := func() { + cancelT() + cancelA() + cancelC() + + if len(config.ChromePath) == 0 { + _ = os.RemoveAll(userDataDir) //nolint:errcheck + } + } + + return ctx, cancel, nil +} + +func supressWelcomeFlag() []chromedp.ExecAllocatorOption { + return []chromedp.ExecAllocatorOption{ + chromedp.Flag("no-first-run", true), + chromedp.Flag("no-default-browser-check", true), + } +} + +func debuggerAddrFlag(config Config) []chromedp.ExecAllocatorOption { + port := strconv.Itoa(config.Port) + if config.Port == 0 { + port = getRandomPort() + } + + return []chromedp.ExecAllocatorOption{ + chromedp.Flag("remote-debugging-host", "127.0.0.1"), + chromedp.Flag("remote-debugging-port", port), + } +} + +func localeFlag() chromedp.ExecAllocatorOption { + lang := "en-US" + if tag, err := locale.Detect(); err != nil && len(tag.String()) > 0 { + lang = tag.String() + } + + return chromedp.Flag("lang", lang) +} + +func noSandboxFlag(config Config) []chromedp.ExecAllocatorOption { + var opts []chromedp.ExecAllocatorOption + + if config.NoSandbox { + opts = append(opts, + chromedp.Flag("no-sandbox", true), + chromedp.Flag("test-type", true)) + } + + return opts +} + +func logLevelFlag(config Config) chromedp.ExecAllocatorOption { + return chromedp.Flag("log-level", strconv.Itoa(config.LogLevel)) +} + +func getRandomPort() string { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err == nil { + addr := l.Addr().String() + _ = l.Close() //nolint:errcheck,gosec + + return strings.Split(addr, ":")[1] + } + + return "42069" +} diff --git a/chrome_windows_test.go b/chrome_windows_test.go new file mode 100644 index 0000000..be3e10d --- /dev/null +++ b/chrome_windows_test.go @@ -0,0 +1,73 @@ +package chromedpundetected + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/chromedp/chromedp" + "github.com/hashicorp/go-multierror" +) + +var n = 3 + +func TestChromedpundetected(t *testing.T) { + testRun(t, + n, + NewConfig( + WithTimeout(20*time.Second), + WithHeadless(), + ), + func(ctx context.Context) error { + return chromedp.Run(ctx, + chromedp.Navigate("https://nowsecure.nl"), + chromedp.WaitVisible(`//div[@class="hystericalbg"]`), + ) + }, + ) + + t.Logf("Undetected!") +} + +func testRun(t *testing.T, n int, cfg Config, run func(ctx context.Context) error) { + var gerr error + var success bool + + // Attempt to run the tests multiple times, as in CI they are unstable. + for i := 0; i < n; i++ { + t.Logf("Attempt %d/%d", i+1, n) + + ctx, cancel, err := New(cfg) + if err != nil { + gerr = multierror.Append(gerr, fmt.Errorf("create context: %w", err)) + continue + } + + if err = run(ctx); err != nil { + gerr = multierror.Append(gerr, fmt.Errorf("chromedp run: %w", err)) + + // Close Chrome instance. + ctxN, cancelN := context.WithTimeout(ctx, time.Second*10) + if err := chromedp.Cancel(ctxN); err != nil { + t.Logf("failed to cancel: %v", err) + } + cancelN() + cancel() + + continue + } + + success = true + if gerr != nil { + t.Logf("attempt %d, errors: %s", i+1, gerr.Error()) + } + + cancel() + break + } + + if !success { + t.Fatal(gerr) + } +} diff --git a/config.go b/config.go index 6222419..dd55f13 100644 --- a/config.go +++ b/config.go @@ -17,7 +17,7 @@ type Option func(*Config) // Config is a undetected Chrome config. type Config struct { - // Ctx is the base context to use. By default a background context will be used. + // Ctx is the base context to use. By default, a background context will be used. Ctx context.Context `json:"-" yaml:"-"` // ContextOptions are chromedp context option. diff --git a/display_unix.go b/display_unix.go new file mode 100644 index 0000000..6509f6d --- /dev/null +++ b/display_unix.go @@ -0,0 +1,165 @@ +//go:build unix + +package chromedpundetected + +import ( + "bufio" + "errors" + "fmt" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + "syscall" + "time" + + "golang.org/x/exp/slog" +) + +// Errors. +var ( + ErrXvfbNotFound = errors.New("xvfb not found. Please install (Linux only)") +) + +// frameBuffer controls an X virtual frame buffer running as a background +// process. +type frameBuffer struct { + // Display is the X11 display number that the Xvfb process is hosting + // (without the preceding colon). + Display string + + // AuthPath is the path to the X11 authorization file that permits X clients + // to use the X server. This is typically provided to the client via the + // XAUTHORITY environment variable. + AuthPath string + + cmd *exec.Cmd +} + +// newFrameBuffer starts an X virtual frame buffer running in the background. +// FrameBufferOptions may be populated to change the behavior of the frame buffer. +func newFrameBuffer(screenSize string) (*frameBuffer, error) { //nolint:funlen + if err := exec.Command("which", "Xvfb").Run(); err != nil { + return nil, ErrXvfbNotFound + } + + pipeReader, pipeWriter, err := os.Pipe() + if err != nil { + return nil, err + } + + defer func() { + if err = pipeReader.Close(); err != nil { + slog.Error("failed to close pipe reader", err) + } + }() + + authPath, err := tempFile("chromedp-xvfb") + if err != nil { + return nil, err + } + + // Xvfb will print the display on which it is listening to file descriptor 3, + // for which we provide a pipe. + arguments := []string{"-displayfd", "3", "-nolisten", "tcp"} + + if screenSize != "" { + screenSizeExpression := regexp.MustCompile(`^\d+x\d+(?:x\d+)$`) + if !screenSizeExpression.MatchString(screenSize) { + return nil, fmt.Errorf("invalid screen size: expected 'WxH[xD]', got %q", screenSize) + } + + arguments = append(arguments, "-screen", "0", screenSize) + } + + xvfb := exec.Command("Xvfb", arguments...) + xvfb.ExtraFiles = []*os.File{pipeWriter} + xvfb.Env = append(xvfb.Env, "XAUTHORITY="+authPath) + + if xvfb.SysProcAttr == nil { + xvfb.SysProcAttr = new(syscall.SysProcAttr) + } + + xvfb.SysProcAttr.Pdeathsig = syscall.SIGKILL + + if err := xvfb.Start(); err != nil { + return nil, err + } + + if err := pipeWriter.Close(); err != nil { + return nil, err + } + + type resp struct { + display string + err error + } + + ch := make(chan resp) + + go func() { + bufr := bufio.NewReader(pipeReader) + s, err := bufr.ReadString('\n') + ch <- resp{s, err} + }() + + var display string + select { + case resp := <-ch: + if resp.err != nil { + return nil, resp.err + } + + display = strings.TrimSpace(resp.display) + if _, err := strconv.Atoi(display); err != nil { + return nil, errors.New("xvfb did not print the display number") + } + + case <-time.After(10 * time.Second): + return nil, errors.New("timeout waiting for Xvfb") + } + + xauth := exec.Command("xauth", "generate", ":"+display, ".", "trusted") //nolint:gosec + xauth.Env = append(xauth.Env, "XAUTHORITY="+authPath) + // Make this conditional? + xauth.Stderr = os.Stderr + xauth.Stdout = os.Stdout + + if err := xauth.Run(); err != nil { + return nil, err + } + + return &frameBuffer{display, authPath, xvfb}, nil +} + +// Stop kills the background frame buffer process and removes the X +// authorization file. +func (f frameBuffer) Stop() error { + if err := f.cmd.Process.Kill(); err != nil { + return err + } + + _ = os.Remove(f.AuthPath) //nolint:errcheck + + if err := f.cmd.Wait(); err != nil && err.Error() != "signal: killed" { + return err + } + + return nil +} + +func tempFile(pattern string) (string, error) { + tempFile, err := os.CreateTemp("", pattern) + if err != nil { + return "", err + } + + fileName := tempFile.Name() + + if err := tempFile.Close(); err != nil { + return "", err + } + + return fileName, nil +}