Merge pull request #33511 from nextcloud/feat/extract_gps_from_exif

Extract GPS data from EXIF

apps/files/lib/Command/Scan.php

@@ -37,8 +37,11 @@ use OC\Core\Command\Base;
 use OC\Core\Command\InterruptedException;
 use OC\DB\Connection;
 use OC\DB\ConnectionAdapter;
+use OCP\Files\File;
 use OC\ForbiddenException;
+use OC\Metadata\MetadataManager;
 use OCP\EventDispatcher\IEventDispatcher;
+use OCP\Files\IRootFolder;
 use OCP\Files\Mount\IMountPoint;
 use OCP\Files\NotFoundException;
 use OCP\Files\StorageNotAvailableException;
@@ -51,19 +54,22 @@ use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 
 class Scan extends Base {
+	private IUserManager $userManager;
+	protected float $execTime = 0;
+	protected int $foldersCounter = 0;
+	protected int $filesCounter = 0;
+	private IRootFolder $root;
+	private MetadataManager $metadataManager;
 
-	/** @var IUserManager $userManager */
-	private $userManager;
-	/** @var float */
-	protected $execTime = 0;
-	/** @var int */
-	protected $foldersCounter = 0;
-	/** @var int */
-	protected $filesCounter = 0;
-
-	public function __construct(IUserManager $userManager) {
+	public function __construct(
+		IUserManager $userManager,
+		IRootFolder $rootFolder,
+		MetadataManager $metadataManager
+	) {
 		$this->userManager = $userManager;
 		parent::__construct();
+		$this->root = $rootFolder;
+		$this->metadataManager = $metadataManager;
 	}
 
 	protected function configure() {
@@ -83,6 +89,12 @@ class Scan extends Base {
 				InputArgument::OPTIONAL,
 				'limit rescan to this path, eg. --path="/alice/files/Music", the user_id is determined by the path and the user_id parameter and --all are ignored'
 			)
+			->addOption(
+				'generate-metadata',
+				null,
+				InputOption::VALUE_NONE,
+				'Generate metadata for all scanned files'
+			)
 			->addOption(
 				'all',
 				null,
@@ -106,21 +118,26 @@ class Scan extends Base {
 			);
 	}
 
-	protected function scanFiles($user, $path, OutputInterface $output, $backgroundScan = false, $recursive = true, $homeOnly = false) {
+	protected function scanFiles(string $user, string $path, bool $scanMetadata, OutputInterface $output, bool $backgroundScan = false, bool $recursive = true, bool $homeOnly = false): void {
 		$connection = $this->reconnectToDatabase($output);
 		$scanner = new \OC\Files\Utils\Scanner(
 			$user,
 			new ConnectionAdapter($connection),
-			\OC::$server->query(IEventDispatcher::class),
+			\OC::$server->get(IEventDispatcher::class),
 			\OC::$server->get(LoggerInterface::class)
 		);
 
 		# check on each file/folder if there was a user interrupt (ctrl-c) and throw an exception
-
-		$scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function ($path) use ($output) {
+		$scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function (string $path) use ($output, $scanMetadata) {
 			$output->writeln("\tFile\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE);
 			++$this->filesCounter;
 			$this->abortIfInterrupted();
+			if ($scanMetadata) {
+				$node = $this->root->get($path);
+				if ($node instanceof File) {
+					$this->metadataManager->generateMetadata($node, false);
+				}
+			}
 		});
 
 		$scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', function ($path) use ($output) {
@@ -197,7 +214,7 @@ class Scan extends Base {
 			++$user_count;
 			if ($this->userManager->userExists($user)) {
 				$output->writeln("Starting scan for user $user_count out of $users_total ($user)");
-				$this->scanFiles($user, $path, $output, $input->getOption('unscanned'), !$input->getOption('shallow'), $input->getOption('home-only'));
+				$this->scanFiles($user, $path, $input->getOption('generate-metadata'), $output, $input->getOption('unscanned'), !$input->getOption('shallow'), $input->getOption('home-only'));
 				$output->writeln('', OutputInterface::VERBOSITY_VERBOSE);
 			} else {
 				$output->writeln("<error>Unknown user $user_count $user</error>");
@@ -291,7 +308,7 @@ class Scan extends Base {
 	protected function formatExecTime() {
 		$secs = round($this->execTime);
 		# convert seconds into HH:MM:SS form
-		return sprintf('%02d:%02d:%02d', (int)($secs / 3600), ( (int)($secs / 60) % 60), $secs % 60);
+		return sprintf('%02d:%02d:%02d', (int)($secs / 3600), ((int)($secs / 60) % 60), $secs % 60);
 	}
 
 	protected function reconnectToDatabase(OutputInterface $output): Connection {

lib/private/Metadata/FileMetadataMapper.php

@@ -3,6 +3,7 @@
 declare(strict_types=1);
 /**
  * @copyright Copyright 2022 Carl Schwan <carl@carlschwan.eu>
+ * @copyright Copyright 2022 Louis Chmn <louis@chmn.me>
  * @license AGPL-3.0-or-later
  *
  * This code is free software: you can redistribute it and/or modify
@@ -24,6 +25,7 @@ namespace OC\Metadata;
 use OCP\AppFramework\Db\DoesNotExistException;
 use OCP\AppFramework\Db\MultipleObjectsReturnedException;
 use OCP\AppFramework\Db\QBMapper;
+use OCP\AppFramework\Db\Entity;
 use OCP\DB\Exception;
 use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\IDBConnection;
@@ -102,4 +104,45 @@ class FileMetadataMapper extends QBMapper {
 
 		$qb->executeStatement();
 	}
+
+	/**
+	 * Updates an entry in the db from an entity
+	 *
+	 * @param Entity $entity the entity that should be created
+	 * @return Entity the saved entity with the set id
+	 * @throws Exception
+	 * @throws \InvalidArgumentException if entity has no id
+	 */
+	public function update(Entity $entity): Entity {
+		if (!($entity instanceof FileMetadata)) {
+			throw new \Exception("Entity should be a FileMetadata entity");
+		}
+
+		// entity needs an id
+		$id = $entity->getId();
+		if ($id === null) {
+			throw new \InvalidArgumentException('Entity which should be updated has no id');
+		}
+
+		// entity needs an group_name
+		$groupName = $entity->getGroupName();
+		if ($groupName === null) {
+			throw new \InvalidArgumentException('Entity which should be updated has no group_name');
+		}
+
+		$idType = $this->getParameterTypeForProperty($entity, 'id');
+		$groupNameType = $this->getParameterTypeForProperty($entity, 'groupName');
+		$metadataValue = $entity->getMetadata();
+		$metadataType = $this->getParameterTypeForProperty($entity, 'metadata');
+
+		$qb = $this->db->getQueryBuilder();
+
+		$qb->update($this->tableName)
+			->set('metadata', $qb->createNamedParameter($metadataValue, $metadataType))
+			->where($qb->expr()->eq('id', $qb->createNamedParameter($id, $idType)))
+			->andWhere($qb->expr()->eq('group_name', $qb->createNamedParameter($groupName, $groupNameType)))
+			->executeStatement();
+
+		return $entity;
+	}
 }

lib/private/Metadata/MetadataManager.php

@@ -21,27 +21,19 @@ namespace OC\Metadata;
 
 use OC\Metadata\Provider\ExifProvider;
 use OCP\Files\File;
-use OCP\IConfig;
-use Psr\Log\LoggerInterface;
 
 class MetadataManager implements IMetadataManager {
 	/** @var array<string, IMetadataProvider> */
 	private array $providers;
 	private array $providerClasses;
 	private FileMetadataMapper $fileMetadataMapper;
-	private IConfig $config;
-	private LoggerInterface $logger;
 
 	public function __construct(
-		FileMetadataMapper $fileMetadataMapper,
-		IConfig $config,
-		LoggerInterface $logger
+		FileMetadataMapper $fileMetadataMapper
 	) {
 		$this->providers = [];
 		$this->providerClasses = [];
 		$this->fileMetadataMapper = $fileMetadataMapper;
-		$this->config = $config;
-		$this->logger = $logger;
 
 		// TODO move to another place, where?
 		$this->registerProvider(ExifProvider::class);

lib/private/Metadata/Provider/ExifProvider.php

@@ -1,23 +1,66 @@
 <?php
 
+declare(strict_types=1);
+/**
+ * @copyright Copyright 2022 Carl Schwan <carl@carlschwan.eu>
+ * @copyright Copyright 2022 Louis Chmn <louis@chmn.me>
+ * @license AGPL-3.0-or-later
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program. If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
 namespace OC\Metadata\Provider;
 
 use OC\Metadata\FileMetadata;
 use OC\Metadata\IMetadataProvider;
 use OCP\Files\File;
+use Psr\Log\LoggerInterface;
 
 class ExifProvider implements IMetadataProvider {
+	private LoggerInterface $logger;
+
+	public function __construct(
+		LoggerInterface $logger
+	) {
+		$this->logger = $logger;
+	}
+
 	public static function groupsProvided(): array {
-		return ['size'];
+		return ['size', 'gps'];
 	}
 
 	public static function isAvailable(): bool {
 		return extension_loaded('exif');
 	}
 
+	/** @return array{'gps': FileMetadata, 'size': FileMetadata} */
 	public function execute(File $file): array {
+		$exifData = [];
 		$fileDescriptor = $file->fopen('rb');
-		$data = exif_read_data($fileDescriptor, 'COMPUTED', true);
+
+		$data = null;
+		try {
+			// Needed to make reading exif data reliable.
+			// This is to trigger this condition: https://github.com/php/php-src/blob/d64aa6f646a7b5e58359dc79479860164239580a/main/streams/streams.c#L710
+			// But I don't understand why 1 as a special meaning.
+			// Revert right after reading the exif data.
+			$oldBufferSize = stream_set_chunk_size($fileDescriptor, 1);
+			$data = exif_read_data($fileDescriptor, 'ANY_TAG', true);
+			stream_set_chunk_size($fileDescriptor, $oldBufferSize);
+		} catch (\Exception $ex) {
+			$this->logger->warning("Couldn't extract metadata for ".$file->getId(), ['exception' => $ex]);
+		}
 
 		$size = new FileMetadata();
 		$size->setGroupName('size');
@@ -31,29 +74,65 @@ class ExifProvider implements IMetadataProvider {
 					'width' => $sizeResult[0],
 					'height' => $sizeResult[1],
 				]);
+
+				$exifData['size'] = $size;
 			}
+		} elseif (array_key_exists('COMPUTED', $data)) {
+			if (array_key_exists('Width', $data['COMPUTED']) && array_key_exists('Height', $data['COMPUTED'])) {
+				$size->setMetadata([
+					'width' => $data['COMPUTED']['Width'],
+					'height' => $data['COMPUTED']['Height'],
+				]);
 
-			return [
-				'size' => $size,
-			];
+				$exifData['size'] = $size;
+			}
 		}
 
-		if (array_key_exists('COMPUTED', $data)
-			&& array_key_exists('Width', $data['COMPUTED'])
-			&& array_key_exists('Height', $data['COMPUTED'])
+		if ($data && array_key_exists('GPS', $data)
+			&& array_key_exists('GPSLatitude', $data['GPS']) && array_key_exists('GPSLatitudeRef', $data['GPS'])
+			&& array_key_exists('GPSLongitude', $data['GPS']) && array_key_exists('GPSLongitudeRef', $data['GPS'])
 		) {
-			$size->setMetadata([
-				'width' => $data['COMPUTED']['Width'],
-				'height' => $data['COMPUTED']['Height'],
+			$gps = new FileMetadata();
+			$gps->setGroupName('gps');
+			$gps->setId($file->getId());
+			$gps->setMetadata([
+				'latitude' => $this->gpsDegreesToDecimal($data['GPS']['GPSLatitude'], $data['GPS']['GPSLatitudeRef']),
+				'longitude' => $this->gpsDegreesToDecimal($data['GPS']['GPSLongitude'], $data['GPS']['GPSLongitudeRef']),
 			]);
+
+			$exifData['gps'] = $gps;
 		}
 
-		return [
-			'size' => $size,
-		];
+		return $exifData;
 	}
 
 	public static function getMimetypesSupported(): string {
 		return '/image\/.*/';
 	}
+
+	/**
+	 * @param array|string $coordinates
+	 */
+	private static function gpsDegreesToDecimal($coordinates, ?string $hemisphere): float {
+		if (is_string($coordinates)) {
+			$coordinates = array_map("trim", explode(",", $coordinates));
+		}
+
+		if (count($coordinates) !== 3) {
+			throw new \Exception('Invalid coordinate format: ' . json_encode($coordinates));
+		}
+
+		[$degrees, $minutes, $seconds] = array_map(function (string $rawDegree) {
+			$parts = explode('/', $rawDegree);
+
+			if ($parts[1] === '0') {
+				return 0;
+			}
+
+			return floatval($parts[0]) / floatval($parts[1] ?? 1);
+		}, $coordinates);
+
+		$sign = ($hemisphere === 'W' || $hemisphere === 'S') ? -1 : 1;
+		return $sign * ($degrees + $minutes / 60 + $seconds / 3600);
+	}
 }