Add .ignore support in the scanner (#679)

INSTALLING.md

@@ -6,7 +6,7 @@
    [`.env`](https://raw.githubusercontent.com/zoriya/Kyoo/master/.env.example) files
 3. Fill the `.env` file with your configuration options
 4. Look at [Hardware Acceleration section](#Hardware-Acceleration) if you need it
-5. Look at [Custom Volumes](#Custom-Volumes) if you need it,
+5. Look at [FAQ](#FAQ) if you need it,
 6. Run `docker compose up -d` and see kyoo at `http://localhost:8901`
 
 # Installing
@@ -91,7 +91,9 @@ You can also add `COMPOSE_PROFILES=nvidia` to your `.env` instead of adding the
 Note that most nvidia cards have an artificial limit on the number of encodes. You can confirm your card limit [here](https://developer.nvidia.com/video-encode-and-decode-gpu-support-matrix-new).
 This limit can also be removed by applying an [unofficial patch](https://github.com/keylase/nvidia-patch) to you driver.
 
-# Custom volumes
+# FAQ
+
+## Custom volumes
 
 To customize volumes, you can edit the `docker-compose.yml` manually.
 
@@ -120,6 +122,18 @@ You can also edit the volume definition to use advanced volume drivers if you ne
 
 Don't forget to **also edit the scanner's volumes** if you edit the transcoder's volume.
 
+## Ignoring Directories
+Kyoo supports excluding specific directories from scanning and monitoring by detecting the presence of a `.ignore` file. When a directory contains a `.ignore` file, Kyoo will recursively exclude that directory and all its contents from processing.
+
+Example:
+To exclude `/media/extras/**`, add a `.ignore` file:
+```bash
+touch /media/extras/.ignore
+```
+Kyoo will skip `/media/extras` and its contents in all future scans and monitoring events.
+
 # OpenID Connect
 
 Kyoo supports OpenID Connect (OIDC) for authentication. Please refer to the [OIDC.md](OIDC.md) file for more information.
+
+<!-- vim: set wrap: -->

scanner/scanner/monitor.py

@@ -1,5 +1,5 @@
 from logging import getLogger
-from os.path import isdir
+from os.path import isdir, dirname, exists, join
 from watchfiles import awatch, Change
 from .publisher import Publisher
 from .scanner import scan, get_ignore_pattern
@@ -8,16 +8,33 @@
 logger = getLogger(__name__)
 
 
+def is_ignored_path(path: str) -> bool:
+	"""Check if the path is within a directory that contains a `.ignore` file."""
+	current_path = path
+	while current_path != "/":  # Traverse up to the root directory
+		if exists(join(current_path, ".ignore")):
+			return True
+		current_path = dirname(current_path)
+	return False
+
+
 async def monitor(path: str, publisher: Publisher, client: KyooClient):
 	ignore_pattern = get_ignore_pattern()
 	async for changes in awatch(path, ignore_permission_denied=True):
 		for event, file in changes:
+			# Check for ignore conditions
+			if is_ignored_path(file):
+				logger.info(
+					"Ignoring event %s for file %s (due to .ignore file)", event, file
+				)
+				continue
 			if ignore_pattern and ignore_pattern.match(file):
 				logger.info(
 					"Ignoring event %s for file %s (due to IGNORE_PATTERN)", event, file
 				)
 				continue
-			logger.info("Change %s occured for file %s", event, file)
+
+			logger.info("Change %s occurred for file %s", event, file)
 			match event:
 				case Change.added if isdir(file):
 					await scan(file, publisher, client)
@@ -28,4 +45,4 @@ async def monitor(path: str, publisher: Publisher, client: KyooClient):
 				case Change.modified:
 					pass
 				case _:
-					logger.warn("Unknown file event %s (for file %s)", event, file)
+					logger.warning("Unknown file event %s (for file %s)", event, file)

scanner/scanner/scanner.py

@@ -11,12 +11,11 @@
 
 
 def get_ignore_pattern():
+	"""Compile ignore pattern from environment variable."""
 	try:
 		pattern = os.environ.get("LIBRARY_IGNORE_PATTERN")
-		if pattern:
-			return re.compile(pattern)
-		return None
-	except Exception as e:
+		return re.compile(pattern) if pattern else None
+	except re.error as e:
 		logger.error(f"Invalid ignore pattern. Ignoring. Error: {e}")
 		return None
 
@@ -25,32 +24,51 @@ async def scan(
 	path_: Optional[str], publisher: Publisher, client: KyooClient, remove_deleted=False
 ):
 	path = path_ or os.environ.get("SCANNER_LIBRARY_ROOT", "/video")
+	logger.info("Starting scan at %s. This may take some time...", path)
 
-	logger.info("Starting the scan. It can take some times...")
 	ignore_pattern = get_ignore_pattern()
+	if ignore_pattern:
+		logger.info(f"Applying ignore pattern: {ignore_pattern}")
+
+	registered = set(await client.get_registered_paths())
+	videos = set()
+
+	for dirpath, dirnames, files in os.walk(path):
+		# Skip directories with a `.ignore` file
+		if ".ignore" in files:
+			dirnames.clear()  # Prevents os.walk from descending into this directory
+			continue
+
+		for file in files:
+			file_path = os.path.join(dirpath, file)
+			# Apply ignore pattern, if any
+			if ignore_pattern and ignore_pattern.match(file_path):
+				continue
+			videos.add(file_path)
 
-	registered = await client.get_registered_paths()
-	videos = [
-		os.path.join(dir, file) for dir, _, files in os.walk(path) for file in files
-	]
-	if ignore_pattern is not None:
-		logger.info(f"Ignoring with pattern {ignore_pattern}")
-		videos = [p for p in videos if not ignore_pattern.match(p)]
-	to_register = [p for p in videos if p not in registered]
+	to_register = videos - registered
+	to_delete = registered - videos if remove_deleted else set()
+
+	if not any(to_register) and any(to_delete) and len(to_delete) == len(registered):
+		logger.warning("All video files are unavailable. Check your disks.")
+		return
+
+	# delete stale files before creating new ones to prevent potential conflicts
+	if to_delete:
+		logger.info("Removing %d stale files.", len(to_delete))
+		await asyncio.gather(*[publisher.delete(path) for path in to_delete])
+
+	if to_register:
+		logger.info("Found %d new files to register.", len(to_register))
+		await asyncio.gather(*[publisher.add(path) for path in to_register])
 
 	if remove_deleted:
-		deleted = [x for x in registered if x not in videos]
-		logger.info("Found %d stale files to remove.", len(deleted))
-		if len(deleted) != len(registered):
-			await asyncio.gather(*map(publisher.delete, deleted))
-		elif len(deleted) > 0:
-			logger.warning("All video files are unavailable. Check your disks.")
-
-		issues = await client.get_issues()
-		for x in issues:
-			if x not in videos:
-				await client.delete_issue(x)
-
-	logger.info("Found %d new files (counting non-video files)", len(to_register))
-	await asyncio.gather(*map(publisher.add, to_register))
+		issues = set(await client.get_issues())
+		issues_to_delete = issues - videos
+		if issues_to_delete:
+			logger.info("Removing %d stale issues.", len(issues_to_delete))
+			await asyncio.gather(
+				*[client.delete_issue(issue) for issue in issues_to_delete]
+			)
+
 	logger.info("Scan finished for %s.", path)