usertrap: disable syscall patching when ptraced

Workaround the issue in #12266.

FUTURE_COPYBARA_INTEGRATE_REVIEW=#12325 from trail-of-forks:ptrace-issue 3c52fed
PiperOrigin-RevId: 840814557

Author: ret2libc

pkg/sentry/platform/systrap/usertrap/usertrap_amd64.go

@@ -193,6 +193,22 @@ func (s *State) PatchSyscall(ctx context.Context, ac *arch.Context64, mm memoryM
 		return fmt.Errorf("no task found")
 	}
 
+	// Skip syscall patching when the task is being ptraced, because
+	// single-stepping and other debugger features are incompatible with
+	// the "syshandler" routine used to handle patched syscalls (see
+	// syshandler_amd64.S). This incompatibility can result in inconsistent
+	// process states and failures (e.g. SIGSEGV).
+	// TODO(gvisor.dev/issue/11649): for a full fix we'd need to roll back
+	//     existing patched syscalls, in case the traced program was patched
+	//     before being traced (e.g. PTRACE_ATTACH on an already running
+	//     process).
+	if task.Tracer() != nil {
+		if s.nextTrap > 0 {
+			ctx.Warningf("LIKELY ERROR: Attached tracer to process with patched syscalls (traps %d)! Systrap is not fully compatible with ptrace/debuggers, program may die unexpectedly soon!", s.nextTrap)
+		}
+		return nil
+	}
+
 	s.mu.Lock()
 	defer s.mu.Unlock()
 

test/runner/main.go

@@ -723,6 +723,10 @@ func isWarning(line string) bool {
 	// Caused by properties of the host that runsc doesn't necessarily control.
 	case strings.Contains(line, "Host limit is lower than recommended"):
 
+	// TODO(gvisor.dev/issue/11649): Systrap needs to roll back created
+	// patches for traced procs.
+	case strings.Contains(line, "LIKELY ERROR: Attached tracer to process with patched syscalls"):
+
 	case *save:
 		// Ignore these warnings for S/R tests as we try to delete the sandbox
 		// after the sandbox has exited and before attempting to restore it.