diff --git a/gazelle/python/generate.go b/gazelle/python/generate.go
index 6fa6252ebb..90e06a1546 100644
--- a/gazelle/python/generate.go
+++ b/gazelle/python/generate.go
@@ -593,26 +593,53 @@ func (py *Python) getRulesWithInvalidSrcs(args language.GenerateArgs, validFiles
 		validFilesMap[file] = struct{}{}
 	}
 
+	// allFilesMap extends validFilesMap with all regular files on disk.
+	// py_binary uses validFilesMap (main modules + generated files), while py_library
+	// and py_test use allFilesMap since any file is a valid src for them.
+	allFilesMap := make(map[string]struct{}, len(validFilesMap)+len(args.RegularFiles))
+	for file := range validFilesMap {
+		allFilesMap[file] = struct{}{}
+	}
+	for _, file := range args.RegularFiles {
+		allFilesMap[file] = struct{}{}
+	}
+
 	isTarget := func(src string) bool {
 		return strings.HasPrefix(src, "@") || strings.HasPrefix(src, "//") || strings.HasPrefix(src, ":")
 	}
 	for _, existingRule := range args.File.Rules {
-		if !kindMatches(args.Config, existingRule, pyBinaryKind) {
+		var matchedKind string
+		var filesMap map[string]struct{}
+		if kindMatches(args.Config, existingRule, pyBinaryKind) {
+			matchedKind = pyBinaryKind
+			filesMap = validFilesMap
+		} else if kindMatches(args.Config, existingRule, pyLibraryKind) {
+			matchedKind = pyLibraryKind
+			filesMap = allFilesMap
+		} else if kindMatches(args.Config, existingRule, pyTestKind) {
+			matchedKind = pyTestKind
+			filesMap = allFilesMap
+		} else {
+			continue
+		}
+
+		srcs := existingRule.AttrStrings("srcs")
+		if len(srcs) == 0 {
 			continue
 		}
 		var hasValidSrcs bool
-		for _, src := range existingRule.AttrStrings("srcs") {
+		for _, src := range srcs {
 			if isTarget(src) {
 				hasValidSrcs = true
 				break
 			}
-			if _, ok := validFilesMap[src]; ok {
+			if _, ok := filesMap[src]; ok {
 				hasValidSrcs = true
 				break
 			}
 		}
 		if !hasValidSrcs {
-			invalidRules = append(invalidRules, newTargetBuilder(pyBinaryKind, existingRule.Name(), "", "", nil, false).build())
+			invalidRules = append(invalidRules, newTargetBuilder(matchedKind, existingRule.Name(), "", "", nil, false).build())
 		}
 	}
 	return invalidRules
diff --git a/gazelle/python/testdata/remove_invalid_library/BUILD.in b/gazelle/python/testdata/remove_invalid_library_package_mode/BUILD.in
similarity index 100%
rename from gazelle/python/testdata/remove_invalid_library/BUILD.in
rename to gazelle/python/testdata/remove_invalid_library_package_mode/BUILD.in
diff --git a/gazelle/python/testdata/remove_invalid_library/BUILD.out b/gazelle/python/testdata/remove_invalid_library_package_mode/BUILD.out
similarity index 100%
rename from gazelle/python/testdata/remove_invalid_library/BUILD.out
rename to gazelle/python/testdata/remove_invalid_library_package_mode/BUILD.out
diff --git a/gazelle/python/testdata/remove_invalid_library/README.md b/gazelle/python/testdata/remove_invalid_library_package_mode/README.md
similarity index 100%
rename from gazelle/python/testdata/remove_invalid_library/README.md
rename to gazelle/python/testdata/remove_invalid_library_package_mode/README.md
diff --git a/gazelle/python/testdata/remove_invalid_library/WORKSPACE b/gazelle/python/testdata/remove_invalid_library_package_mode/WORKSPACE
similarity index 100%
rename from gazelle/python/testdata/remove_invalid_library/WORKSPACE
rename to gazelle/python/testdata/remove_invalid_library_package_mode/WORKSPACE
diff --git a/gazelle/python/testdata/remove_invalid_library/my_test.py b/gazelle/python/testdata/remove_invalid_library_package_mode/my_test.py
similarity index 100%
rename from gazelle/python/testdata/remove_invalid_library/my_test.py
rename to gazelle/python/testdata/remove_invalid_library_package_mode/my_test.py
diff --git a/gazelle/python/testdata/remove_invalid_library/others/BUILD.in b/gazelle/python/testdata/remove_invalid_library_package_mode/others/BUILD.in
similarity index 100%
rename from gazelle/python/testdata/remove_invalid_library/others/BUILD.in
rename to gazelle/python/testdata/remove_invalid_library_package_mode/others/BUILD.in
diff --git a/gazelle/python/testdata/remove_invalid_library/others/BUILD.out b/gazelle/python/testdata/remove_invalid_library_package_mode/others/BUILD.out
similarity index 100%
rename from gazelle/python/testdata/remove_invalid_library/others/BUILD.out
rename to gazelle/python/testdata/remove_invalid_library_package_mode/others/BUILD.out
diff --git a/gazelle/python/testdata/remove_invalid_library/test.yaml b/gazelle/python/testdata/remove_invalid_library_package_mode/test.yaml
similarity index 100%
rename from gazelle/python/testdata/remove_invalid_library/test.yaml
rename to gazelle/python/testdata/remove_invalid_library_package_mode/test.yaml
diff --git a/gazelle/python/testdata/remove_invalid_per_file/BUILD.in b/gazelle/python/testdata/remove_invalid_per_file/BUILD.in
new file mode 100644
index 0000000000..7f387ab6f9
--- /dev/null
+++ b/gazelle/python/testdata/remove_invalid_per_file/BUILD.in
@@ -0,0 +1,36 @@
+load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
+
+# gazelle:python_generation_mode file
+
+py_library(
+    name = "bar",
+    srcs = ["bar.py"],
+    visibility = ["//:__subpackages__"],
+)
+
+py_library(
+    name = "deleted_lib",
+    srcs = ["deleted.py"],
+    visibility = ["//:__subpackages__"],
+)
+
+py_binary(
+    name = "remove_invalid_per_file_bin",
+    srcs = ["__main__.py"],
+    visibility = ["//:__subpackages__"],
+)
+
+py_binary(
+    name = "deleted_bin",
+    srcs = ["deleted_bin.py"],
+)
+
+py_test(
+    name = "bar_test",
+    srcs = ["bar_test.py"],
+)
+
+py_test(
+    name = "deleted_test",
+    srcs = ["deleted_test.py"],
+)
diff --git a/gazelle/python/testdata/remove_invalid_per_file/BUILD.out b/gazelle/python/testdata/remove_invalid_per_file/BUILD.out
new file mode 100644
index 0000000000..db62ff2df4
--- /dev/null
+++ b/gazelle/python/testdata/remove_invalid_per_file/BUILD.out
@@ -0,0 +1,21 @@
+load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
+
+# gazelle:python_generation_mode file
+
+py_library(
+    name = "bar",
+    srcs = ["bar.py"],
+    visibility = ["//:__subpackages__"],
+)
+
+py_test(
+    name = "bar_test",
+    srcs = ["bar_test.py"],
+)
+
+py_binary(
+    name = "remove_invalid_per_file_bin",
+    srcs = ["__main__.py"],
+    main = "__main__.py",
+    visibility = ["//:__subpackages__"],
+)
diff --git a/gazelle/python/testdata/remove_invalid_per_file/WORKSPACE b/gazelle/python/testdata/remove_invalid_per_file/WORKSPACE
new file mode 100644
index 0000000000..faff6af87a
--- /dev/null
+++ b/gazelle/python/testdata/remove_invalid_per_file/WORKSPACE
@@ -0,0 +1 @@
+# This is a Bazel workspace for the Gazelle test data.
diff --git a/gazelle/python/testdata/remove_invalid_per_file/__main__.py b/gazelle/python/testdata/remove_invalid_per_file/__main__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/gazelle/python/testdata/remove_invalid_per_file/bar.py b/gazelle/python/testdata/remove_invalid_per_file/bar.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/gazelle/python/testdata/remove_invalid_per_file/bar_test.py b/gazelle/python/testdata/remove_invalid_per_file/bar_test.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/gazelle/python/testdata/remove_invalid_per_file/test.yaml b/gazelle/python/testdata/remove_invalid_per_file/test.yaml
new file mode 100644
index 0000000000..ed97d539c0
--- /dev/null
+++ b/gazelle/python/testdata/remove_invalid_per_file/test.yaml
@@ -0,0 +1 @@
+---