ALib C++ Library
Library Version: 2412 R0
Documentation generated by doxygen
Loading...
Searching...
No Matches
fscanner.cpp
1// #################################################################################################
2// ALib C++ Library
3//
4// Copyright 2013-2024 A-Worx GmbH, Germany
5// Published under 'Boost Software License' (a free software lbicense, see LICENSE.txt)
6// #################################################################################################
8
11
13#include "alib/alox.hpp"
14
16
17#if !DOXYGEN
18
19
20
21using namespace alib::lang::system;
22
23namespace alib::files { namespace {
24
25 // forward declaration of startScan()
26 bool startScan( FTree& tree,
27 PathString realPath,
28 ScanParameters& params,
29 FInfo::DirectorySums& parentSums,
30 std::vector<ResultsPaths>& resultPaths
31 IF_ALIB_THREADS(, SharedLock* lock ) );
32
33 // scan parameters used with startScan to evaluate directory entries
34 ScanParameters paramsPathOnly( nullptr, ScanParameters::SymbolicLinks::DONT_RESOLVE, 0, true, true );
35} // namespace alib::files[::anonymous]
36
37#if ALIB_DEBUG
39 A_CHAR(" {:ta h{2,r} on{10,r} gn{10,r} s(IEC){10,r} dm qqq FxFa (rd{3r}' D' rf{3r}' F' re{2r}' EA' rb{2r}'BL) 'nf l}");
40#endif
41
42} // namespace [alib::files]
43
44//--------------------------------------------------------------------------------------------------
45//--- scanFilePosix
46//--------------------------------------------------------------------------------------------------
47#if ALIB_FILES_SCANNER_IMPL == ALIB_FILES_SCANNER_POSIX
48# include <unistd.h>
49# if defined(__linux__)
50# include <asm/unistd.h>
51# endif
52# include <dirent.h>
53# if defined(__linux__)
54# include <linux/stat.h>
55# include <sys/stat.h>
56# else
57# include <sys/stat.h>
58# endif
59# if !defined(__APPLE__)
60# include <sys/sysmacros.h>
61# else
62# include <sys/types.h>
63# endif
64# include <pwd.h>
65# include <fcntl.h>
66# include <pwd.h>
67# include <grp.h>
68
69#if ALIB_DEBUG
70# define DBG_CHECKERRNO \
71 ALIB_ASSERT_WARNING(errno == 0, "FILES", "Errno set ({}){!Q}.", \
72 errno, SystemErrors(errno) ) \
73 errno= 0;
74# define DBG_CHECKERRNO_WITH_PATH \
75 ALIB_ASSERT_WARNING(errno == 0, "FILES", "Errno set ({}){!Q}. Current path: {}", \
76 errno, SystemErrors(errno), actPath ) \
77 errno= 0;
78#else
79# define DBG_CHECKERRNO
80# define DBG_CHECKERRNO_WITH_PATH
81#endif
82
83// Since Kernel 4.11 Linux/glibc has "statx". We use this is available on the current platform.
84#if defined(__NR_statx)
85# define TMP_STATX_AVAILABLE 1
86# define STATMEMBER(Name) stats.stx_ ## Name
87# define STAT_DEV_MAJOR stats.stx_dev_major
88# define STAT_DEV_MINOR stats.stx_dev_minor
89#else
90# define TMP_STATX_AVAILABLE 0
91# define STATMEMBER(Name) stats.st_ ## Name
92# define STAT_DEV_MAJOR major(stats.st_dev)
93# define STAT_DEV_MINOR minor(stats.st_dev)
94#endif
95
96namespace alib::files { namespace {
97void scanFilePosix( DIR* pxDir,
98 FTree::Cursor& node,
99 const CPathString& nameOrFullPath, // if full path, this has the same buffer as actPath!
100 unsigned int depth,
101 ScanParameters& params,
102 uint64_t currentDevice,
103 FInfo::DirectorySums& parentSums ,
104 Path& actPath,
105 std::vector<ResultsPaths>& resultPaths
106 IF_ALIB_THREADS(, SharedLock* lock) )
107{
108 ALIB_ASSERT_ERROR( actPath.CharAtStart()== DIRECTORY_SEPARATOR
109 && ( actPath.Length()==1
110 || actPath.CharAtEnd() != DIRECTORY_SEPARATOR )
112 DIRECTORY_SEPARATOR).Append(DIRECTORY_SEPARATOR)) < 0 ,
113 "FILES","Given path not absolute or ending with '{}': {}", DIRECTORY_SEPARATOR, actPath )
114 ALIB_DBG( Path dbgActFile;
115 if( actPath.Buffer() == nameOrFullPath.Buffer() )
116 dbgActFile << nameOrFullPath;
117 else
118 {
119 dbgActFile << actPath;
120 if(dbgActFile.Length()>1)
121 dbgActFile << DIRECTORY_SEPARATOR;
122 dbgActFile << nameOrFullPath;
123 } )
124 Log_Prune(
125 LocalAllocator<1> verboseAllocator;
126 BoxesMA verboseLogables( verboseAllocator);
127 int verboseLoggers;
128 Log_IsActive(verboseLoggers, Verbosity::Verbose)
129 if( verboseLoggers )
130 {
131 verboseLogables.Add("{!AWidth:>} ");
132 if( &params == &paramsPathOnly )
133 verboseLogables.Add("PO"); // 'Path Only'
134 else
135 {
136 auto& depthString= *verboseAllocator().New<String128>();
137 depthString << depth << DIRECTORY_SEPARATOR
138 << ( params.MaxDepth < std::numeric_limits<unsigned int>::max()
139 ? String128(params.MaxDepth)
140 : String(A_CHAR("M")) );
141 verboseLogables.Add(depthString);
142 }
143
144 verboseLogables.Add(files::DBG_FILES_SCAN_VERBOSE_LOG_FORMAT, File(node) );
145 }
146 )
147 ALIB_ASSERT_WARNING(errno == 0, "FILES", "Errno set ({}){!Q} with current file: {}",
148 errno, SystemErrors(errno), dbgActFile )
149 ALIB_DBG( errno= 0;)
150
151 auto& value = *node;
152 auto oldQuality= value.Quality();
153
154 ALIB_STRINGS_TO_NARROW(nameOrFullPath, nNameOrFullPath, 512)
155
156 // ------------------------------ get stats? ------------------------------
157 if( value.Quality() == FInfo::Qualities::NONE
158 || ( value.Quality() == FInfo::Qualities::STATS
159 && params.LinkTreatment != ScanParameters::SymbolicLinks::DONT_RESOLVE ) )
160 {
161 value.SetQuality(FInfo::Qualities::STATS);
162 Path symLinkDest;
163 Path symLinkDestReal;
164
165 // read base stats
166 ALIB_DBG( errno= 0;)
167 #if TMP_STATX_AVAILABLE
168 struct statx stats;
169 int statResult= statx( pxDir ? dirfd(pxDir) : 0,
170 nNameOrFullPath,
171 AT_STATX_DONT_SYNC | AT_NO_AUTOMOUNT | AT_SYMLINK_NOFOLLOW,
172 STATX_BASIC_STATS | STATX_BTIME,
173 &stats );
174
175 #else
176 struct stat stats;
177 int statResult= pxDir ? fstatat(dirfd(pxDir), nNameOrFullPath, &stats,
178 AT_SYMLINK_NOFOLLOW
179 #if !defined(__APPLE__)
180 | AT_NO_AUTOMOUNT
181 #endif
182 )
183 : lstat ( nNameOrFullPath, &stats );
184 #endif
185 if( statResult )
186 {
187 ALIB_ASSERT_WARNING( errno != ENOENT, "FILES", "File does not exist (anymore) while stating {!Q}",
188 dbgActFile )
189 ALIB_ASSERT_WARNING( errno == ENOENT, "FILES", "Unknown error ({}) {!Q} while stating file {!Q}",
190 errno, SystemErrors(errno), dbgActFile )
191 value.SetQuality(errno == ENOENT ? FInfo::Qualities::NOT_EXISTENT
193 ALIB_DBG( errno= 0;)
194 goto APPLY_POST_RECURSION_FILTER;
195 }
196 DBG_CHECKERRNO_WITH_PATH
197
198 // check filesystem type (artificial fs & mount point)
199 {
200 uint64_t device= (uint64_t(STAT_DEV_MAJOR) << 32L) + STAT_DEV_MINOR;
201 if( currentDevice == 0) currentDevice= device;
202 else if( currentDevice != device) { currentDevice= device; value.SetCrossingFS(); }
203 }
204
205 if( STAT_DEV_MAJOR == 0 // artificial?
206 && STAT_DEV_MINOR != 35 ) // tmpfs included, not considered artificial!
207 value.SetArtificialFS();
208
209 //------------ is symlink? ------------
210 bool origFileIsSymlink= (STATMEMBER(mode) & S_IFMT) == S_IFLNK;
211 if( origFileIsSymlink
213 {
214 value.SetQuality( FInfo::Qualities::RESOLVED );
215
216 // 1. Read plain symlink target (only to be attached to the entry)
217 ALIB_STRINGS_TO_NARROW(symLinkDest, nSymLinkDest, 512)
218 ssize_t cntChars= pxDir ? readlinkat( dirfd(pxDir), nNameOrFullPath, nSymLinkDest.VBuffer(), PATH_MAX)
219 : readlink ( nNameOrFullPath, nSymLinkDest.VBuffer(), PATH_MAX);
220
221 if (cntChars == -1) switch(errno)
222 {
223 case EACCES: value.SetQuality(FInfo::Qualities::NO_ACCESS_SL); ALIB_DBG(errno= 0;)
224 goto ABORT_SYMLINK;
225
226 case ENOENT: value.SetQuality(FInfo::Qualities::NO_ACCESS_SL);
227 ALIB_ASSERT_ERROR(STAT_DEV_MAJOR == 0, "FILES",
228 "Posix raised ({}) {!Q} on reading a symbolic link which is not located on an "
229 "artificial filesystem (like /proc). File:{!Q}",
230 errno, SystemErrors(errno), dbgActFile ) ALIB_DBG(errno= 0;)
231 goto ABORT_SYMLINK;
232
233 default: value.SetQuality(FInfo::Qualities::UNKNOWN_ERROR);
234 ALIB_ERROR("FILES", "Posix raised ({}) {!Q} on reading symbolic link {!Q}",
235 errno, SystemErrors(errno), dbgActFile ) ALIB_DBG(errno= 0;)
236 goto ABORT_SYMLINK;
237 }
238 nSymLinkDest.SetLength(cntChars);
240 symLinkDest.Reset(nSymLinkDest);
241 #endif
242
243 // 2. Read symlink's real target path (fully and recursively translated)
244 ALIB_STRING_RESETTER(actPath);
245 if( pxDir )
246 actPath << DIRECTORY_SEPARATOR << nameOrFullPath;
247 errno= 0;
248 ALIB_STRINGS_TO_NARROW(actPath , nActPath , 512)
249 ALIB_STRINGS_TO_NARROW(symLinkDestReal, nSymLinkDestReal, 512)
250 *nSymLinkDestReal.VBuffer()= '\0';
251 if(! realpath(nActPath.Terminate(), nSymLinkDestReal.VBuffer() ) ) switch (errno)
252 { // The named file does not exist.
253 case ENOENT: if( *nSymLinkDestReal.VBuffer() != '\0')
254 nSymLinkDestReal.DetectLength();
255 value.SetQuality(FInfo::Qualities::BROKEN_LINK); ALIB_DBG(errno= 0;)
256 goto ABORT_SYMLINK;
257 case ELOOP: value.SetQuality(FInfo::Qualities::CIRCULAR_LINK); ALIB_DBG(errno= 0;)
258 goto ABORT_SYMLINK;
259 // this might happen with strange system files
260 case EACCES: value.SetQuality(FInfo::Qualities::NO_ACCESS_SL_TARGET); ALIB_DBG(errno= 0;)
261 goto ABORT_SYMLINK;
262 default: ALIB_ERROR("FILES", "Posix raised ({}) {!Q} on resolving symbolic link {!Q}",
263 errno, SystemErrors(errno), dbgActFile ) ALIB_DBG(errno= 0;)
264 goto ABORT_SYMLINK;
265 }
266 nSymLinkDestReal.DetectLength();
267
268 ALIB_DBG( if( errno == EINVAL) errno= 0;) // this happens, even though realpath() above returned 'OK'
269 DBG_CHECKERRNO_WITH_PATH
270 ALIB_ASSERT_ERROR( Path::IsAbsolute(symLinkDestReal), "FILES",
271 "Real path is not absolute: ", nSymLinkDestReal )
272
273 // 3. get resolved status
274 DBG_CHECKERRNO_WITH_PATH
275 #if TMP_STATX_AVAILABLE
276 statResult= statx( 0,
277 nSymLinkDestReal.Terminate(),
278 AT_STATX_DONT_SYNC | AT_NO_AUTOMOUNT | AT_SYMLINK_NOFOLLOW,
279 STATX_ALL,
280 &stats );
281 #else
282 statResult= stat(nSymLinkDestReal.Terminate(), &stats );
283 #endif
284 DBG_CHECKERRNO_WITH_PATH
285 #if ALIB_CHARACTERS_WIDE
286 symLinkDestReal.Reset(nSymLinkDestReal);
287 #endif
288
289 if(statResult == -1 )
290 {
292 if(errno) switch( SystemErrors(errno) )
293 { case SystemErrors::enoent: value.SetQuality(FInfo::Qualities::BROKEN_LINK);
294 ALIB_DBG(errno= 0;)
295 goto APPLY_POST_RECURSION_FILTER;
296 default: ALIB_WARNING("FILES",
297 "Unhandled error code invoking 'stat()' on resolved symbolic link: {} ({!Q})\n"
298 " Symbolic link target: {!Q}", errno, SystemErrors(errno), dbgActFile )
299 ALIB_DBG(errno= 0;)
300 value.SetQuality(FInfo::Qualities::UNKNOWN_ERROR);
301 goto APPLY_POST_RECURSION_FILTER;
302 }
304 }
305
306 // check for target artificial fs
307 if( STAT_DEV_MAJOR == 0 // artificial?
308 && STAT_DEV_MINOR != 35 ) // tempfs included!
309 value.SetTargetArtificialFS();
310
311 } // if is symlink && resolve symlinks
312
313 ABORT_SYMLINK:
314 DBG_CHECKERRNO_WITH_PATH
315
316 // =========================================================================================
317 // ================================== Copy Stats ==================================
318 // =========================================================================================
319 // 1. type
320 {
322 auto posixType= STATMEMBER(mode) & S_IFMT;
323 if( origFileIsSymlink )
324 {
325 type= posixType == S_IFDIR ? FInfo::Types::SYMBOLIC_LINK_DIR
327 }
328 else switch(STATMEMBER(mode) & S_IFMT )
329 {
330 case S_IFLNK : type= FInfo::Types::SYMBOLIC_LINK; ALIB_ERROR( "FILES", "Impossible")
331 break;
332 case S_IFBLK : type= FInfo::Types::BLOCK ; break;
333 case S_IFCHR : type= FInfo::Types::CHARACTER ; break;
334 case S_IFDIR : type= FInfo::Types::DIRECTORY ; break;
335 case S_IFIFO : type= FInfo::Types::FIFO ; break;
336 case S_IFREG : type= FInfo::Types::REGULAR ; break;
337 case S_IFSOCK: type= FInfo::Types::SOCKET ; break;
338 default: ALIB_ERROR("FILES", "Internal error. 'unknown' file type can't happen. File: {!Q}",
339 dbgActFile ) break;
340 }
341 value.SetType( type );
342 }
343
344 // 2. perms
345 value.SetPerms( FInfo::Permissions(STATMEMBER(mode) & int32_t(FInfo::Permissions::MASK)) );
346
347 // 3. timestamps
348 {
349 #if defined(__APPLE__)
350 # define st_mtime_name STATMEMBER(mtimespec)
351 # define st_ctime_name STATMEMBER(ctimespec)
352 # define st_atime_name STATMEMBER(atimespec)
353 #else
354 # if TMP_STATX_AVAILABLE
355 # define st_mtime_name STATMEMBER(mtime)
356 # define st_ctime_name STATMEMBER(ctime)
357 # define st_atime_name STATMEMBER(atime)
358 # define st_btime_name STATMEMBER(btime)
359 # else
360 # define st_mtime_name STATMEMBER(mtim)
361 # define st_ctime_name STATMEMBER(ctim)
362 # define st_atime_name STATMEMBER(atim)
363 # endif
364 #endif
365 DateTime dt;
366 dt.Import(
367 std::chrono::system_clock::time_point {
368 std::chrono::duration_cast<std::chrono::system_clock::duration>(
369 std::chrono::seconds {st_mtime_name.tv_sec }
370 + std::chrono::nanoseconds{st_mtime_name.tv_nsec} ) } );
371 value.SetMDate(dt);
372
373 dt.Import(
374 std::chrono::system_clock::time_point {
375 std::chrono::duration_cast<std::chrono::system_clock::duration>(
376 std::chrono::seconds {st_ctime_name.tv_sec }
377 + std::chrono::nanoseconds{st_ctime_name.tv_nsec} ) } );
378 value.SetCDate(dt);
379
380 dt.Import(
381 std::chrono::system_clock::time_point {
382 std::chrono::duration_cast<std::chrono::system_clock::duration>(
383 std::chrono::seconds {st_atime_name.tv_sec }
384 + std::chrono::nanoseconds{st_atime_name.tv_nsec} ) } );
385 value.SetADate(dt);
386
387 #if TMP_STATX_AVAILABLE
388 if( STATMEMBER(mask) & STATX_BTIME ) // file systems supports "btime"?
389 {
390 dt.Import(
391 std::chrono::system_clock::time_point {
392 std::chrono::duration_cast<std::chrono::system_clock::duration>(
393 std::chrono::seconds {st_btime_name.tv_sec }
394 + std::chrono::nanoseconds{st_btime_name.tv_nsec} ) } );
395 value.SetBDate(dt);
396 }
397 else
398 {
399 // use smallest of other times for "btime"
400 auto btime= value.MDate();
401 if( btime > value.CDate() ) btime= value.CDate();
402 if( btime > value.ADate() ) btime= value.ADate();
403 value.SetBDate( btime );
404
405 }
406 #else
407 // use smallest of other times for "btime"
408 auto btime= value.MDate();
409 if( btime > value.CDate() ) btime= value.CDate();
410 if( btime > value.ADate() ) btime= value.ADate();
411 value.SetBDate( btime );
412 #endif
413
414
415 #undef st_mtime_name
416 #undef st_ctime_name
417 #undef st_atime_name
418 }
419
420 // 4. size
421 value.SetSize( uinteger(STATMEMBER(size) ) );
422
423 // 5. uid/gid
424 value.SetOwner( STATMEMBER(uid) );
425 value.SetGroup( STATMEMBER(gid) );
426
427 // 6. qty of symlinks
428 value.SetQtyHardlinks( STATMEMBER(nlink) );
429
430 // 7. Add extended information
431 if( oldQuality < FInfo::Qualities::STATS
432 && (value.IsDirectory() || symLinkDest.IsNotEmpty()) ) {
433 IF_ALIB_THREADS( if (lock) lock->Acquire(ALIB_CALLER_PRUNED); )
434 File(node).GetFTree().AllocateExtendedInfo( node, symLinkDest, symLinkDestReal );
435 IF_ALIB_THREADS( if (lock) lock->Release(ALIB_CALLER_PRUNED); )
436 }
437
438 } // if scan stats (quality was just path)
439
440 DBG_CHECKERRNO_WITH_PATH
441
442 // Count broken link.
443 if(value.Quality() == FInfo::Qualities::BROKEN_LINK)
444 {
445 ++parentSums.QtyErrsBrokenLink;
446 goto APPLY_POST_RECURSION_FILTER;
447 }
448
449 // =============================================================================================
450 // ============================= recursion with directories? ============================
451 // =============================================================================================
452 if( !value.IsDirectory()
453 || value.Quality() >= FInfo::Qualities::RECURSIVE )
454 goto APPLY_POST_RECURSION_FILTER;
455
456 // stop recursion due to artificial fs?
457 if( value.IsArtificialFS() && !params.IncludeArtificialFS )
458 {
459 Log_Prune( if( verboseLogables.Size() ) verboseLogables.Add(" NO_AFS"); )
460 value.SetQuality( FInfo::Qualities::NO_AFS );
461 goto APPLY_POST_RECURSION_FILTER;
462 }
463
464 // stop recursion due to crossing filesystem?
465 if( value.IsCrossingFS() && !params.CrossFileSystems )
466 {
467 Log_Prune( if( verboseLogables.Size() ) verboseLogables.Add(" NOT_CROSSING_FS"); )
468 value.SetQuality( FInfo::Qualities::NOT_CROSSING_FS );
469 goto APPLY_POST_RECURSION_FILTER;
470 }
471
472 // stop recursion due to max depth?
473 if( depth >= params.MaxDepth )
474 {
475 Log_Prune( if( verboseLogables.Size() && (&params != &paramsPathOnly) ) verboseLogables.Add(" MAX_DEPTH_REACHED"); )
476 value.SetQuality( FInfo::Qualities::MAX_DEPTH_REACHED );
477 ++parentSums.QtyStopsOnMaxDepth;
478 goto APPLY_POST_RECURSION_FILTER;
479 }
480
481 // stop recursion due to filter
482 if( depth > 0
483 && params.DirectoryFilterPreRecursion
484 && !params.DirectoryFilterPreRecursion->Includes( node, actPath ) )
485 {
486 Log_Prune( if( verboseLogables.Size() ) verboseLogables.Add(" FILTERED(Pre)"); )
487 goto APPLY_POST_RECURSION_FILTER;
488 }
489
490 // mark as recursively scanned
491 value.SetQuality( FInfo::Qualities::RECURSIVE );
492
493 // SYMLINK RECURSION
494 if ( value.Type() == FInfo::Types::SYMBOLIC_LINK_DIR )
495 {
496 if( params.LinkTreatment != ScanParameters::SymbolicLinks::RECURSIVE
497 || value.IsArtificialFS() ) // never recurse with symlinks RESIDING on artificial fs!
498 {
499 value.SetQuality( FInfo::Qualities::NOT_FOLLOWED );
500 goto APPLY_POST_RECURSION_FILTER;
501 }
502
503 if( value.TargetIsArtificialFS() && !params.IncludeArtificialFS )
504 {
505 value.SetQuality( FInfo::Qualities::NO_AFS );
506 goto APPLY_POST_RECURSION_FILTER;
507 }
508
509 // recurse into symlink target
510 FInfo::DirectorySums childSums;
511 if( startScan( node.Tree<FTree>(), value.GetRealLinkTarget(), params, childSums,
512 resultPaths IF_ALIB_THREADS(,lock) ) )
513 value.SetQuality(FInfo::Qualities::DUPLICATE);
514 value.SetSums( childSums );
515 parentSums+= childSums;
516
517 goto APPLY_POST_RECURSION_FILTER;
518 }
519
520 // DIRECTORY RECURSION
521 {ALIB_STRING_RESETTER( actPath );
522 if( pxDir == nullptr )
523 {
524 ALIB_ASSERT_ERROR( actPath.Buffer() == nameOrFullPath.Buffer(), "FILES", "Internal error" )
525 actPath.SetLength(nameOrFullPath.Length());
526 }
527 else
528 {
529 if( actPath.Length() > 1 ) actPath << DIRECTORY_SEPARATOR;
530 actPath << nameOrFullPath;
531 }
532
533 errno= 0;
534 int fd;
535 if( pxDir)
536 fd= openat( dirfd(pxDir), nNameOrFullPath, O_RDONLY | O_DIRECTORY );
537 else
538 {
539 ALIB_STRINGS_TO_NARROW(actPath, nActPath, 512)
540 fd= open( nActPath , O_RDONLY | O_DIRECTORY );
541 }
542
543 if (fd != -1) // success?
544 {
545 DBG_CHECKERRNO_WITH_PATH
546 FInfo::DirectorySums subSums;
547 DIR* childDir = fdopendir(fd);
548 for(;;)
549 {
550 errno= 0;
551 dirent* pxEntry = readdir(childDir);
552 if( pxEntry == nullptr )
553 {
554 switch(errno)
555 {
556 // possible errors (according to documentation):
557 // EOVERFLOW One of the values in the structure to be returned cannot be represented correctly.
558 // EBADF The dirp argument does not refer to an open directory stream.
559 // ENOENT The current position of the directory stream is invalid.
560 case 0: break;
561 case EACCES: value.SetQuality(FInfo::Qualities::NO_ACCESS_DIR);
562 break;
563 case EINVAL: value.SetQuality( FInfo::Qualities::NO_ACCESS_DIR);
564 ALIB_ASSERT_ERROR(major(currentDevice) == 0, "FILES",
565 "Posix raised ({}) {!Q} on reading a directory which is not located on an "
566 "artificial filesystem (like /proc). File:{!Q}",
567 errno, SystemErrors(errno), dbgActFile )
568 break;
569 default: value.SetQuality(FInfo::Qualities::UNKNOWN_ERROR);
570 ALIB_ERROR("FILES", "Posix raised ({}) {!Q} on reading directory {!Q}",
571 errno, SystemErrors(errno), dbgActFile )
572 break;
573 }
574 errno= 0;
575 break;
576 }
577
578 // skip "." and ".."
579 if( pxEntry->d_name[0] == '.'
580 && ( pxEntry->d_name[1] == '\0'
581 || ( pxEntry->d_name[1] == '.'
582 && pxEntry->d_name[2] == '\0' ) ) )
583 continue;
584
585 //----- recursive call -----
586 auto childNode= node;
587#if ALIB_CHARACTERS_WIDE
588 Path childName(const_cast<const char*>(&pxEntry->d_name[0]));
589#else
590 const CString childName(const_cast<const char*>(&pxEntry->d_name[0]));
591#endif
592 IF_ALIB_THREADS( if (lock) lock->Acquire(ALIB_CALLER_PRUNED); )
593 childNode.GoToCreateChildIfNotExistent( childName );
594 IF_ALIB_THREADS( if (lock) lock->Release(ALIB_CALLER_PRUNED); )
595 scanFilePosix( childDir, childNode, childName,
596 depth + 1, params, currentDevice, subSums, actPath,
597 resultPaths IF_ALIB_THREADS(,lock) );
598 } // dir entry loop
599 closedir(childDir);
600 DBG_CHECKERRNO_WITH_PATH
601
602 // previously scanned in lower quality?
603 if( oldQuality != FInfo::Qualities::NONE )
604 {
605 FTree::FixSums( node );
606 parentSums+= value.Sums();
607 }
608 else
609 {
610 value.SetSums(subSums);
611 parentSums+= subSums;
612 }
613 ALIB_DBG( errno= 0;)
614 goto APPLY_POST_RECURSION_FILTER;
615 } // success opening director
616
617 // error with recursion
618 ALIB_ASSERT_ERROR(errno != ENOTDIR, "FILES",
619 "Internal error opening directory. This must never happen")
620
621
622 ALIB_WARNINGS_ALLOW_SPARSE_ENUM_SWITCH
623 switch (SystemErrors(errno))
624 {
625 case SystemErrors::eacces:
626 ++parentSums.QtyErrsAccess;
627 value.SetQuality( FInfo::Qualities::NO_ACCESS_DIR );
628 errno= 0;
629 break;
630
631 default:
632 ALIB_ERROR("FILES", "Unknown error {}({!Q}) while opening directory {!Q}",
633 errno, SystemErrors(errno), actPath)
634 value.SetQuality( FInfo::Qualities::UNKNOWN_ERROR );
635 break;
636 }
637 }
638
639 // =============================================================================================
640 // ==================== Apply Post Filter and remove empty directories ===================
641 // =============================================================================================
642 APPLY_POST_RECURSION_FILTER:
643 // delete node only if this was a new scan.
644 // It must not be deleted if this node was created as a path.
645 if( oldQuality == FInfo::Qualities::NONE )
646 {
647 if ( value.IsDirectory() )
648 {
649 if( depth > 0
650 && ( ( params.DirectoryFilterPostRecursion
651 && !params.DirectoryFilterPostRecursion->Includes(node, actPath ) )
652 || ( params.RemoveEmptyDirectories
653 && value.Sums().Count() == 0 )
654 ) )
655 {
656 Log_Prune( if( verboseLogables.Size() ) { verboseLogables.Add(" FILTERED(Post)");
657 Log_Verbose( verboseLogables )
658 verboseLogables.clear(); } )
659 parentSums-= value.Sums();
660 if( params.RemoveEmptyDirectories )
661 {
662 File file(node);
663 node.Tree<FTree>().Notify( FTreeListener::Event::DeleteNode,
664 file,
665 IF_ALIB_THREADS(lock,)
666 actPath );
667 node.Delete();
668 return;
669 }
670
671 // Notify deletion of all children.
672 auto it= node.FirstChild();
673 while ( it.IsValid() )
674 {
675 File file(node);
676 node.Tree<FTree>().Notify( FTreeListener::Event::DeleteNode,
677 file,
678 IF_ALIB_THREADS(lock,)
679 actPath );
680 it.GoToNextSibling();
681 }
682
683 // do not return here. Still count the type below
684 node.DeleteChildren();
685 }
686
687 }
688 else
689 {
690 if ( params.FileFilter
691 && !params.FileFilter->Includes(node, actPath ) )
692 {
693 Log_Prune( if( verboseLogables.Size() ) { verboseLogables.Add(" FILTERED(Post)");
694 Log_Verbose( verboseLogables ) } )
695 IF_ALIB_THREADS( if (lock) lock->Acquire(ALIB_CALLER_PRUNED); )
696 node.Delete();
697 IF_ALIB_THREADS( if (lock) lock->Release(ALIB_CALLER_PRUNED); )
698 return;
699 }
700 }
701 }
702
703 Log_Prune( if( verboseLogables.Size() ) { Log_Verbose( verboseLogables ) } )
704
705 // cnt file type
706 parentSums.Add(value);
707 File file(node);
708 node.Tree<FTree>().Notify( FTreeListener::Event::CreateNode, file, IF_ALIB_THREADS(lock,) actPath );
709
710 ALIB_WARNINGS_RESTORE
711 DBG_CHECKERRNO_WITH_PATH
712} // scanFilePosix()
713
714}} // namespace [alib::files::anonymous]
715#undef DBG_CHECKERRNO_WITH_PATH
716#undef TMP_STATX_AVAILABLE
717#undef STATMEMBER
718
719
720//--------------------------------------------------------------------------------------------------
721//--- UNKNOWN platform, using C++17 filesystem (not all functionality given)
722//--------------------------------------------------------------------------------------------------
723#else
724
725#if ALIB_FILES_FORCE_STD_SCANNER
726# pragma message ("ALIB_FILES_FORCE_STD_SCANNER given. Using std::filesystem for scanning. In file: " __FILE__ )
727#else
728# pragma message ("Unknown Platform. Using std::filesystem for scanning. In file: " __FILE__ )
729#endif
730
731#include "alib/compatibility/std_strings.hpp"
732#include <filesystem>
733namespace fs = std::filesystem;
734
735// Note: MacOS is currently (as of 231210) missing C++ 20 library features in the area of std::clock
736#if ALIB_CPP_STANDARD == 17 || defined(__APPLE__) || defined(__ANDROID_NDK__)
737 namespace
738 {
739 template <typename TP>
740 std::time_t to_time_t(TP tp)
741 {
742 auto sctp = std::chrono::time_point_cast<std::chrono::system_clock::duration>(tp - TP::clock::now()
743 + std::chrono::system_clock::now());
744 return std::chrono::system_clock::to_time_t(sctp);
745 }
746 }
747#endif
748#if ALIB_DEBUG
749# define DBG_CHECKERRNO \
750 ALIB_ASSERT_WARNING(errno == 0, "FILES", "Errno set ({}){!Q}.", \
751 errno, SystemErrors(errno) ) \
752 errno= 0;
753# define DBG_CHECKERRNO_WITH_PATH \
754 ALIB_ASSERT_WARNING(errno == 0, "FILES", "Errno set ({}){!Q}. Current path: {}", \
755 errno, SystemErrors(errno), path.string() ) \
756 errno= 0;
757#else
758# define DBG_CHECKERRNO
759# define DBG_CHECKERRNO_WITH_PATH
760#endif
761
762namespace alib::files { namespace {
763
764void scanFileStdFS( const fs::path& path,
765 FTree::Cursor& node,
766 unsigned int depth,
767 ScanParameters& params,
768 FInfo::DirectorySums& parentSums,
769 std::vector<ResultsPaths>& resultPaths
770 IF_ALIB_THREADS(, SharedLock* lock) )
771{
772#if defined(__MINGW32__)
773 Path pathAsCString(path.c_str());
774 pathAsCString.Terminate();
775#else
776 CPathString pathAsCString(path.c_str());
777#endif
778 const PathSubstring parentPath= pathAsCString.Substring(0, pathAsCString.LastIndexOf(DIRECTORY_SEPARATOR));
779
780 #if !defined(_WIN32)
781 ALIB_ASSERT_ERROR( pathAsCString.CharAtStart()== DIRECTORY_SEPARATOR
782 && ( pathAsCString.Length()==1
783 || pathAsCString.CharAtEnd() != DIRECTORY_SEPARATOR)
784 && pathAsCString.IndexOf(NString8(DIRECTORY_SEPARATOR).Append(DIRECTORY_SEPARATOR))<0,
785 "FILES","Given path not absolute or ending with '{}': {}", DIRECTORY_SEPARATOR, pathAsCString )
786 #else
787 ALIB_ASSERT_ERROR( ( ( pathAsCString.CharAt(1)== ':'
788 && pathAsCString.CharAt(2)== DIRECTORY_SEPARATOR
789 && ( pathAsCString.Length()==3
790 || pathAsCString.CharAtEnd() != DIRECTORY_SEPARATOR) )
791
792 || ( pathAsCString.CharAt(0)== DIRECTORY_SEPARATOR
793 && pathAsCString.CharAt(1)== DIRECTORY_SEPARATOR
794 && ( pathAsCString.Length()==2
795 || pathAsCString.CharAtEnd() != DIRECTORY_SEPARATOR) )
796 )
797 && pathAsCString.IndexOf( strings::TLocalString<PathCharType, 8>(
798 DIRECTORY_SEPARATOR).Append(DIRECTORY_SEPARATOR),
799 2 ) < 0,
800 "FILES","Given path not absolute or ending with '{}': {}", DIRECTORY_SEPARATOR, pathAsCString )
801 #endif
802
803
804 Log_Verbose( "[{}] {}/{} {}", &params != &paramsPathOnly ? '>':'P', depth,
805 params.MaxDepth < (std::numeric_limits<unsigned int>::max)()
806 ? String128(params.MaxDepth)
807 : String(A_CHAR("M")),
808 pathAsCString )
809
810 std::error_code errorCode;
811 auto& value = node.Value();
812 auto oldQuality= value.Quality();
813
814 // ------------------------------ get stats? ------------------------------
815 if( value.Quality() == FInfo::Qualities::NONE
816 || ( value.Quality() == FInfo::Qualities::STATS
817 && params.LinkTreatment != ScanParameters::SymbolicLinks::DONT_RESOLVE ) )
818 {
819 value.SetQuality( FInfo::Qualities::STATS );
820 Path symLinkDest;
821 Path symLinkDestReal;
822
823 // read base stats (we have to use symlink_status() which does NOT follow the symlink!)
824 fs::file_status stats= fs::symlink_status(path);
825 ALIB_WARNINGS_ALLOW_SPARSE_ENUM_SWITCH
826 if(errorCode)
827 {
828 ALIB_ERROR("FILES",
829 "Unhandled error code invoking 'fs::symlink_status()': {} ({!Q})\n"
830 " With file: {!Q}",
831 errorCode.value(), errorCode.message(), pathAsCString )
832 ALIB_DBG( errno= 0;)
833 value.SetQuality(FInfo::Qualities::UNKNOWN_ERROR);
834 goto APPLY_POST_RECURSION_FILTER;
835 }
836 ALIB_WARNINGS_RESTORE
837 ALIB_DBG(errno= 0;)
838
839 //------------ is symlink? ------------
840 bool origFileIsSymlink= (stats.type() == fs::file_type::symlink);
841 if( origFileIsSymlink
842 && params.LinkTreatment != ScanParameters::SymbolicLinks::DONT_RESOLVE )
843 {
844 value.SetQuality( FInfo::Qualities::RESOLVED );
845
846 // 1. Read plain symlink target (only to be attached to the entry)
847 fs::path resolved= fs::read_symlink(path, errorCode);
848 if(errorCode)
849 {
850 ALIB_WARNINGS_ALLOW_SPARSE_ENUM_SWITCH
851 switch( SystemErrors(errorCode.value()) )
852 { case SystemErrors::enoent: // happens with /proc files
853 case SystemErrors::eacces: value.SetQuality(FInfo::Qualities::NO_ACCESS_SL);
854 ALIB_DBG(errno= 0;)
855 goto ABORT_SYMLINK;
856 default: ALIB_ERROR("FILES", "Unhandled error code invoking 'fs::read_symlink()': {} ({!Q})\n"
857 " with file: ", errorCode.value(), errorCode.message(), pathAsCString )
858 ALIB_DBG( errno= 0;)
859 value.SetQuality(FInfo::Qualities::UNKNOWN_ERROR);
860 goto APPLY_POST_RECURSION_FILTER;
861 }
862 ALIB_WARNINGS_RESTORE
863 }
864 DBG_CHECKERRNO_WITH_PATH
865 symLinkDest << resolved.c_str();
866
867 // 2. Read symlink's real target path (fully and recursively translated)
868 fs::path realPath;
869 if( resolved.is_absolute() )
870 realPath= fs::canonical(resolved, errorCode);
871 else
872 {
873 symLinkDestReal << pathAsCString;
874 symLinkDestReal.ShortenTo( symLinkDestReal.LastIndexOf(DIRECTORY_SEPARATOR) + 1);
875 symLinkDestReal << symLinkDest;
876 realPath= fs::canonical(fs::path(
877 std::basic_string_view<PathCharType>(symLinkDestReal.Buffer(),
878 size_t(symLinkDestReal.Length()))),
879 errorCode);
880 symLinkDestReal.Reset();
881 }
882 ALIB_DBG(if(errno==EINVAL && !errorCode) errno= 0;) // this happens!, we do not care, but clean up
883 ALIB_DBG(if(errno==ENOENT && !errorCode) errno= 0;)
884
885 ALIB_WARNINGS_ALLOW_SPARSE_ENUM_SWITCH
886 if(errorCode) switch( SystemErrors(errorCode.value()) )
887 { // we ignore this: std::fs would not create the "real path" if the final directory is not accessible.
888 case SystemErrors::eacces: value.SetQuality(FInfo::Qualities::NO_ACCESS_SL_TARGET); ALIB_DBG(errno= 0;) goto ABORT_SYMLINK;
889 case SystemErrors::enoent: value.SetQuality(FInfo::Qualities::BROKEN_LINK); ALIB_DBG(errno= 0;) goto ABORT_SYMLINK;
890 case SystemErrors::eloop: value.SetQuality(FInfo::Qualities::CIRCULAR_LINK); ALIB_DBG(errno= 0;) goto ABORT_SYMLINK;
891 default: ALIB_ERROR("FILES", "Unhandled error code invoking 'fs::canonical()': {} ({!Q})\n"
892 " with file: ", errorCode.value(), errorCode.message(), pathAsCString )
893 goto ABORT_SYMLINK;
894 }
895 ALIB_WARNINGS_RESTORE
896 DBG_CHECKERRNO_WITH_PATH
897 symLinkDestReal << realPath.c_str();
898
899 // 3. get resolved status
900 ALIB_WARNINGS_ALLOW_SPARSE_ENUM_SWITCH
901 auto newStatus= fs::status(path, errorCode);
902 if(!errorCode)
903 {
904 // this happens with strange /proc files...
905 if(newStatus.type() != fs::file_type::unknown)
906 stats= newStatus;
907 }
908 else switch( SystemErrors(errorCode.value()) )
909 { case SystemErrors::eperm: value.SetQuality( FInfo::Qualities::NO_ACCESS ); ALIB_DBG(errno= 0;) goto ABORT_SYMLINK;
910 case SystemErrors::enoent: value.SetQuality( FInfo::Qualities::BROKEN_LINK ); ALIB_DBG(errno= 0;) goto ABORT_SYMLINK;
911 case SystemErrors::eloop: value.SetQuality( FInfo::Qualities::CIRCULAR_LINK); ALIB_DBG(errno= 0;) goto ABORT_SYMLINK;
912 default: ALIB_WARNING("FILES",
913 "Unhandled error code invoking 'directory_entry::status()': {} ({!Q})\n"
914 " With file: {!Q}", errorCode.value(), errorCode.message(), pathAsCString )
915 goto ABORT_SYMLINK;
916 }
917 ALIB_WARNINGS_RESTORE
918
919 // check for target artificial fs
920 // -/- Not available with std::filesystem version
921 }
922
923 ABORT_SYMLINK:
924 DBG_CHECKERRNO_WITH_PATH
925
926 // =========================================================================================
927 // ================================== Copy Stats ==================================
928 // =========================================================================================
929 // 1. type
930 {
931 auto type= FInfo::Types::UNKNOWN_OR_ERROR;
932 if( origFileIsSymlink )
933 {
934 type= is_directory(stats) ? FInfo::Types::SYMBOLIC_LINK_DIR
935 : FInfo::Types::SYMBOLIC_LINK;
936 }
937 else switch( stats.type() )
938 {
939 case fs::file_type::directory: type= FInfo::Types::DIRECTORY ; break;
940 case fs::file_type::regular : type= FInfo::Types::REGULAR ; break;
941 case fs::file_type::symlink : type= FInfo::Types::SYMBOLIC_LINK; break; // for now, this is a file.
942 case fs::file_type::block : type= FInfo::Types::BLOCK ; break;
943 case fs::file_type::character: type= FInfo::Types::CHARACTER ; break;
944 case fs::file_type::fifo : type= FInfo::Types::FIFO ; break;
945 case fs::file_type::socket : type= FInfo::Types::SOCKET ; break;
946
947 case fs::file_type::not_found:
948 value.SetQuality(FInfo::Qualities::UNKNOWN_ERROR);
949 ALIB_WARNING("FILES", "Internal error. 'not found' file type can't happen. File: ", pathAsCString )
950 ALIB_DBG( errno= 0;) goto APPLY_POST_RECURSION_FILTER;
951 case fs::file_type::none :
952 value.SetQuality(FInfo::Qualities::UNKNOWN_ERROR);
953 ALIB_WARNING("FILES", "Internal error. 'none' file type can't happen. File: ", pathAsCString)
954 ALIB_DBG( errno= 0;) goto APPLY_POST_RECURSION_FILTER;
955 case fs::file_type::unknown :
956 value.SetQuality(FInfo::Qualities::UNKNOWN_ERROR);
957 ALIB_WARNING("FILES", "Internal error. Can't happen. File: ", pathAsCString)
958 ALIB_DBG( errno= 0;) goto APPLY_POST_RECURSION_FILTER;
959 default:
960 value.SetQuality(FInfo::Qualities::UNKNOWN_ERROR);
961 ALIB_WARNING("FILES", "Unknown fs::file_status::type '{}' with file {}.", stats.type(), pathAsCString)
962 ALIB_DBG( errno= 0;) goto APPLY_POST_RECURSION_FILTER;
963 }
964 value.SetType( type );
965 }
966
967 // 2. perms
968 value.SetPerms( FInfo::Permissions(int32_t(stats.permissions())) );
969
970 // 3. timestamps
971 // attn: This method always follows symbolic link and uses the target's time
972 // This seems to be a confirmed behavior:
973 // https://stackoverflow.com/questions/50778660/boost-filesystem-how-to-get-last-write-time-for-symlink-without-resolving
974 auto fsTime= std::filesystem::file_time_type(std::filesystem::file_time_type::clock::now());
975 if ( value.Quality() <= FInfo::Qualities::RESOLVED ) // no error
976 {
977 fsTime= fs::last_write_time( path, errorCode );
979 if(errorCode) switch( SystemErrors(errorCode.value()) )
980 { // This happens if with symbolic links that point to nowhere.
981 case SystemErrors::enoent: ALIB_ERROR( "FILES",
982 "Internal error. This should never happen, checked above. "
983 "Undefined system error handling" ) ALIB_DBG( errno= 0;)
984 value.SetQuality(FInfo::Qualities::UNKNOWN_ERROR);
985 break;
986
987 default: ALIB_ERROR( "FILES", "Unhandled error code invoking 'fs::last_write_time()': {} ({!Q})\n"
988 " With file {!Q}.", errorCode.value(), errorCode.message(), pathAsCString )
989 fsTime= (decltype(fsTime)::min)(); ALIB_DBG( errno= 0;)
990 break;
991 }
992 ALIB_WARNINGS_RESTORE
993 }
994
995
996 #if ALIB_CPP_STANDARD == 17 || defined(__APPLE__) || defined(__ANDROID_NDK__)
997 value.SetMDate( DateTime::FromEpochSeconds( to_time_t( fsTime ) ) );
998 #else
999 value.SetMDate( DateTime::FromEpochSeconds( std::chrono::system_clock::to_time_t(
1000 std::chrono::clock_cast<std::chrono::system_clock>(fsTime) ) ) );
1001 #endif
1002 value.SetBDate( value.MDate() );
1003 value.SetCDate( value.MDate() );
1004 value.SetADate( value.MDate() );
1005
1006 // 4. size
1007 errorCode.clear();
1008 value.SetSize( symLinkDest.Length() > 0 ? uinteger(symLinkDest.Length())
1009 : value.Quality() <= FInfo::Qualities::RESOLVED ? uinteger(fs::file_size(path, errorCode))
1010 : 0 );
1011 if( value.Size() == uinteger(-1))
1012 {
1013 value.SetSize(0);
1014 ALIB_WARNINGS_ALLOW_SPARSE_ENUM_SWITCH
1015 switch( SystemErrors(errorCode.value()) )
1016 {
1017 // target is a directory (no error)
1018 case SystemErrors::eisdir:
1019 break;
1020
1021 case SystemErrors::enoent: // this happens if we have a broken symbolic link
1022 ALIB_ASSERT_ERROR( value.Type() == FInfo::Types::SYMBOLIC_LINK
1023 || value.Type() == FInfo::Types::SYMBOLIC_LINK_DIR , "FILES",
1024 "Internal error. This should never happen. Undefined system error handling" )
1025 break;
1026
1027 // size not supported. Happens with sockets, files in /proc, etc
1028 case SystemErrors::eopnotsupp: break;
1029 default: ALIB_ERROR("FILES", "Unhandled error code invoking 'directory_entry::file_size()':{} ({!Q})\n"
1030 " With file {!Q}.", errorCode.value(), errorCode.message(), pathAsCString )
1031 ALIB_DBG( errno= 0;)
1032 break;
1033 }
1034 ALIB_WARNINGS_RESTORE
1035 }
1036
1037 // 5. uid/gid
1038 value.SetOwner( FInfo::UnknownID );
1039 value.SetGroup( FInfo::UnknownID );
1040
1041 // 6. qty of symlinks
1042 uint32_t qtyHardLinks= uint32_t( fs::hard_link_count(path, errorCode ) );
1043 ALIB_WARNINGS_ALLOW_SPARSE_ENUM_SWITCH
1044 if(errorCode)
1045 {
1046 ALIB_MESSAGE("FILES",
1047 "Unhandled error code invoking 'fs::hard_link_count()': {} ({!Q})\n"
1048 " With file: {!Q}",
1049 errorCode.value(), errorCode.message(), pathAsCString )
1050 ALIB_DBG( errno= 0;)
1051 }
1052 ALIB_WARNINGS_RESTORE
1053 value.SetQtyHardlinks( qtyHardLinks );
1054
1055 // 7. Add extended information
1056 if( oldQuality < FInfo::Qualities::STATS
1057 && (value.IsDirectory() || symLinkDest.IsNotEmpty()) ) {
1058 IF_ALIB_THREADS( if (lock) lock->Acquire(ALIB_CALLER_PRUNED); )
1059 File(node).GetFTree().AllocateExtendedInfo( node, symLinkDest, symLinkDestReal );
1060 IF_ALIB_THREADS( if (lock) lock->Release(ALIB_CALLER_PRUNED); )
1061 }
1062
1063 } // if scan stats (quality was just path)
1064
1065 DBG_CHECKERRNO_WITH_PATH
1066
1067 // Count broken link.
1068 if(value.Quality() == FInfo::Qualities::BROKEN_LINK)
1069 {
1070 ++parentSums.QtyErrsBrokenLink;
1071 goto APPLY_POST_RECURSION_FILTER;
1072 }
1073
1074 // ------------------------------ recursion with directories? ------------------------------
1075 if( !value.IsDirectory()
1076 || value.Quality() >= FInfo::Qualities::RECURSIVE )
1077 goto APPLY_POST_RECURSION_FILTER;
1078
1079
1080 // stop recursion due to artificial fs?
1081 // Not supported with std::filesystem!
1082
1083 // stop recursion due to crossing filesystem?
1084 if( value.IsCrossingFS() && !params.CrossFileSystems )
1085 {
1086 value.SetQuality( FInfo::Qualities::NOT_CROSSING_FS );
1087 goto APPLY_POST_RECURSION_FILTER;
1088 }
1089
1090 // stop recursion due to max depth?
1091 if( depth >= params.MaxDepth )
1092 {
1093 value.SetQuality( FInfo::Qualities::MAX_DEPTH_REACHED );
1094 ++parentSums.QtyStopsOnMaxDepth;
1095 goto APPLY_POST_RECURSION_FILTER;
1096 }
1097
1098 // stop recursion due to filter
1099 if( depth > 0
1100 && params.DirectoryFilterPreRecursion
1101 && !params.DirectoryFilterPreRecursion->Includes( node, parentPath ) )
1102 goto APPLY_POST_RECURSION_FILTER;
1103
1104 // mark as recursively scanned
1105 value.SetQuality( FInfo::Qualities::RECURSIVE );
1106
1107 // SYMLINK RECURSION
1108 if ( value.Type() == FInfo::Types::SYMBOLIC_LINK_DIR )
1109 {
1110 if( params.LinkTreatment != ScanParameters::SymbolicLinks::RECURSIVE
1111 || value.IsArtificialFS() ) // never recurse with symlinks RESIDING on artificial fs!
1112 {
1113 value.SetQuality( FInfo::Qualities::NOT_FOLLOWED );
1114 goto APPLY_POST_RECURSION_FILTER;
1115 }
1116 else
1117 {
1118 // recurse into symlink target
1119 FInfo::DirectorySums childSums;
1120 if( startScan( File(node).GetFTree(), value.GetRealLinkTarget(), params, childSums,
1121 resultPaths IF_ALIB_THREADS(,lock) ) )
1122 value.SetQuality(FInfo::Qualities::DUPLICATE);
1123 value.SetSums( childSums );
1124 parentSums+= childSums;
1125 goto APPLY_POST_RECURSION_FILTER;
1126 }
1127 }
1128
1129 // DIRECTORY RECURSION
1130 {
1131 fs::directory_iterator dit= fs::directory_iterator(path, errorCode);
1132 if(!errorCode) // success?
1133 {
1134 FInfo::DirectorySums subSums;
1135 for( const fs::directory_entry& childDir : dit )
1136 {
1137 // recursive call
1138 #if defined(_WIN32)
1139 Path mingwBuf( childDir.path().c_str());
1140 PathSubstring childName(mingwBuf);
1141 #else
1142 NSubstring childName(NCString(childDir.path().c_str()));
1143 #endif
1144 childName.ConsumeChars(childName.LastIndexOf(DIRECTORY_SEPARATOR) + 1);
1145 auto childNode= node;
1146 IF_ALIB_THREADS( if (lock) lock->Acquire(ALIB_CALLER_PRUNED); )
1147 childNode.GoToCreateChildIfNotExistent( childName );
1148 IF_ALIB_THREADS( if (lock) lock->Release(ALIB_CALLER_PRUNED); )
1149 scanFileStdFS( childDir.path(), childNode, depth + 1, params, subSums,
1150 resultPaths IF_ALIB_THREADS(,lock) );
1151 }
1152
1153 // previously scanned in lower quality?
1154 if( oldQuality != FInfo::Qualities::NONE )
1155 {
1156 FTree::FixSums( node );
1157 parentSums+= value.Sums();
1158 }
1159 else
1160 {
1161 value.SetSums(subSums);
1162 parentSums+= subSums;
1163 }
1164 ALIB_DBG( errno= 0;)
1165 goto APPLY_POST_RECURSION_FILTER;
1166 }
1167 }
1168
1169 // error with recursion
1170 ALIB_ASSERT_ERROR(errorCode.value() != ENOTDIR, "FILES",
1171 "Internal error opening directory. This must never happen")
1172
1173 ALIB_WARNINGS_ALLOW_SPARSE_ENUM_SWITCH
1174 if(errorCode) switch (SystemErrors(errorCode.value()))
1175 {
1176 case SystemErrors::einval: // happens with /proc
1177 case SystemErrors::eacces: ++parentSums.QtyErrsAccess;
1178 value.SetQuality( FInfo::Qualities::NO_ACCESS_DIR );
1179 ALIB_DBG( errno= 0;)
1180 goto APPLY_POST_RECURSION_FILTER;
1181
1182 default: value.SetQuality(FInfo::Qualities::UNKNOWN_ERROR);
1183 ALIB_ERROR("FILES", "Unknown error {}({!Q}) while opening directory {!Q}",
1184 errorCode.value(), SystemErrors(errorCode.value()), pathAsCString)
1185 ALIB_DBG( errno= 0;)
1186 goto APPLY_POST_RECURSION_FILTER;
1187 }
1188 ALIB_WARNINGS_RESTORE
1189 ALIB_DBG( errno= 0;)
1190
1191 // ------------------------------------ Apply Filter ------------------------------------------
1192 APPLY_POST_RECURSION_FILTER:
1193 // delete node only if this was a new scan. It must not be deleted if this node was
1194 // created as a path.
1195 if( oldQuality == FInfo::Qualities::NONE )
1196 {
1197 if ( value.IsDirectory() )
1198 {
1199 if( depth > 0
1200 && ( ( params.DirectoryFilterPostRecursion
1201 && !params.DirectoryFilterPostRecursion->Includes(node, parentPath ) )
1202 || ( params.RemoveEmptyDirectories
1203 && value.Sums().Count() == 0 )
1204 ) )
1205 {
1206 parentSums-= value.Sums();
1207 if( params.RemoveEmptyDirectories )
1208 {
1209 File file(node);
1210 node.Tree<FTree>().Notify( FTreeListener::Event::DeleteNode,
1211 file,
1212 IF_ALIB_THREADS( lock, )
1213 parentPath );
1214 node.Delete();
1215 return;
1216 }
1217
1218 // Notify deletion of all children.
1219 auto it= node.FirstChild();
1220 while ( it.IsValid() )
1221 {
1222 File file(node);
1223 node.Tree<FTree>().Notify( FTreeListener::Event::DeleteNode,
1224 file,
1225 IF_ALIB_THREADS( lock, )
1226 parentPath );
1227 it.GoToNextSibling();
1228 }
1229
1230 // do not return here. Still count the type below
1231 node.DeleteChildren();
1232 }
1233
1234 }
1235 else
1236 {
1237 if ( params.FileFilter
1238 && !params.FileFilter->Includes(node, parentPath ) )
1239 {
1240 IF_ALIB_THREADS( if (lock) lock->Acquire(ALIB_CALLER_PRUNED); )
1241 node.Delete();
1242 IF_ALIB_THREADS( if (lock) lock->Release(ALIB_CALLER_PRUNED); )
1243 return;
1244 }
1245 }
1246 }
1247
1248 // cnt file type
1249 parentSums.Add(value);
1250 File file(node);
1251 node.Tree<FTree>().Notify( FTreeListener::Event::CreateNode, file, IF_ALIB_THREADS(lock,) parentPath );
1252
1253} // scanFileStdFS
1254
1255
1256}} // namespace [alib::files::anonymous]
1257
1258#undef DBG_CHECKERRNO_WITH_PATH
1259#endif // std::fs version
1260
1261//--------------------------------------------------------------------------------------------------
1262//--- ALL Platforms
1263//--------------------------------------------------------------------------------------------------
1264namespace alib::files {
1265
1266namespace {
1267
1268// Creates start path nodes and invokes scanFileXXX
1269bool startScan( FTree& tree,
1270 PathString realPath,
1271 ScanParameters& params,
1272 FInfo::DirectorySums& parentSums,
1273 std::vector<ResultsPaths>& resultPaths
1274 IF_ALIB_THREADS(, SharedLock* lock) )
1275{
1276 ALIB_ASSERT_ERROR( Path::IsAbsolute(realPath), "FILES",
1277 "Real path is not absolute: ", realPath )
1278
1279 FTree::Cursor node= tree.Root().AsCursor();
1280#if !defined(_WIN32)
1281 Path path(DIRECTORY_SEPARATOR);
1282
1283 // travel any existing portion of the path
1284 IF_ALIB_THREADS( if (lock) lock->AcquireShared(ALIB_CALLER_PRUNED); )
1285 PathSubstring pathRemainder= node.GoTo( realPath );
1286 IF_ALIB_THREADS( if (lock) lock->ReleaseShared(ALIB_CALLER_PRUNED); )
1287 path << realPath.Substring(1, realPath.Length() - pathRemainder.Length() - 1);
1288#else
1289 Path path;
1290 PathSubstring pathRemainder;
1291 if(realPath.CharAt(1) == ':')
1292 {
1293 path << realPath.Substring(0,3);
1294 IF_ALIB_THREADS( if (lock) lock->Acquire(ALIB_CALLER_PRUNED); )
1295 node.GoToCreateChildIfNotExistent(realPath.Substring(0,2));
1296 pathRemainder= node.GoTo( realPath.Substring(3) );
1297 IF_ALIB_THREADS( if (lock) lock->Release(ALIB_CALLER_PRUNED); )
1298 path << realPath.Substring(3, realPath.Length() - pathRemainder.Length() -3 );
1299 }
1300 else
1301 {
1302 integer serverNameEnd= realPath.IndexOf( DIRECTORY_SEPARATOR, 2);
1303 if( serverNameEnd < 0)
1304 serverNameEnd= realPath.Length();
1305 path << realPath.Substring(0, serverNameEnd);
1306 IF_ALIB_THREADS( if (lock) lock->Acquire(ALIB_CALLER_PRUNED); )
1307 node.GoToCreateChildIfNotExistent(realPath.Substring(2, serverNameEnd - 2));
1308 pathRemainder= node.GoTo( realPath.Substring(serverNameEnd + 1) );
1309 IF_ALIB_THREADS( if (lock) lock->Release(ALIB_CALLER_PRUNED); )
1310 path << realPath.Substring(serverNameEnd, realPath.Length() - pathRemainder.Length() -serverNameEnd );
1311 }
1312
1313#endif
1314
1315 DBG_CHECKERRNO
1316
1317
1318 // existed already?
1319 if( pathRemainder.IsEmpty() )
1320 {
1321 // For directories, call scan just for the case of having 'higher' scan parameters
1322 if( node->IsDirectory())
1323 {
1324
1325 #if ( (defined(__GLIBCXX__) && !defined(__MINGW32__)) \
1326 || defined(__APPLE__) \
1327 || defined(__ANDROID_NDK__) ) && !ALIB_FILES_FORCE_STD_SCANNER
1328
1329 path.Terminate();
1330 CPathString fullPathChildName(path);
1331 path.SetLength(path.LastIndexOf(DIRECTORY_SEPARATOR) );
1332 scanFilePosix( nullptr, node, fullPathChildName, 0, params, 0, parentSums, path,
1333 resultPaths IF_ALIB_THREADS(,lock));
1334 #else
1335 scanFileStdFS( fs::path(std::basic_string_view<PathCharType>(path.Buffer(),
1336 size_t(path.Length()))),
1337 node, 0, params, parentSums, resultPaths IF_ALIB_THREADS(,lock) );
1338 #endif
1339
1340 //resultPaths.emplace_back(ResultsPaths(realPath, node, true));
1341 }
1342
1343 return true;
1344 }
1345
1346 // did not exist already
1347 if( path.Length() > 1 )
1348 path.DeleteEnd<NC>(1);
1349
1350 strings::util::TTokenizer<PathCharType> tknzr( pathRemainder, DIRECTORY_SEPARATOR );
1351 while(tknzr.HasNext())
1352 {
1353 PathString name;
1354 if( path.Length() != 1 )
1355 {
1356 name= tknzr.Next();
1357IF_ALIB_THREADS( if (lock) lock->Acquire(ALIB_CALLER_PRUNED); )
1358 node= node.CreateChild(name);
1359IF_ALIB_THREADS( if (lock) lock->Release(ALIB_CALLER_PRUNED); )
1360 }
1361
1362 bool isLastPathElement= !tknzr.HasNext();
1363 if( isLastPathElement )
1364 parentSums= FInfo::DirectorySums(); // clear the sums, because only the results of the last element are used.
1365
1366 IF_ALIB_THREADS( if (lock) lock->AcquireShared(ALIB_CALLER_PRUNED); )
1367 auto detectNodeDeletion= node.Depth();
1368 IF_ALIB_THREADS( if (lock) lock->ReleaseShared(ALIB_CALLER_PRUNED); )
1369
1370 #if ( (defined(__GLIBCXX__) && !defined(__MINGW32__)) \
1371 || defined(__APPLE__) \
1372 || defined(__ANDROID_NDK__) ) && !ALIB_FILES_FORCE_STD_SCANNER
1373
1374 if( path.IsEmpty() ) path << DIRECTORY_SEPARATOR;
1375 CPathString fullPathChildName;
1376 {
1377 // add node name to existing path and use same buffer for fullPathChildName!
1378 ALIB_STRING_RESETTER( path );
1379 if( path.Length() > 1 ) path << DIRECTORY_SEPARATOR;
1380 path << node.Name();
1381 path.Terminate();
1382 fullPathChildName= path;
1383 }
1384
1385 scanFilePosix( nullptr, node, fullPathChildName,
1386 0, isLastPathElement ? params : paramsPathOnly,
1387 0, parentSums, path, resultPaths IF_ALIB_THREADS(,lock) );
1388 if( fullPathChildName.Length() == 1 ) path.Reset();
1389 else { if(path.Length() > 1) path << DIRECTORY_SEPARATOR; path << name; }
1390 #else
1391 if( path.Length() != 1 ) path << DIRECTORY_SEPARATOR << name;
1392 scanFileStdFS( fs::path(std::basic_string_view<PathCharType>(path.Buffer(),
1393 size_t(path.Length()))),
1394 node, 0,
1395 isLastPathElement ? params : paramsPathOnly,
1396 parentSums, resultPaths IF_ALIB_THREADS(,lock) );
1397 if( path.Length() == 1 ) path.Reset();
1398 #endif
1399
1400 // if the just created node was not deleted during scan, add it to the result list
1401 if( isLastPathElement)
1402 {
1403 IF_ALIB_THREADS( if (lock) lock->AcquireShared(ALIB_CALLER_PRUNED); )
1404 if (detectNodeDeletion == node.Depth() )
1405 resultPaths.insert(resultPaths.begin(), ResultsPaths(realPath, node, false));
1406 IF_ALIB_THREADS( if (lock) lock->ReleaseShared(ALIB_CALLER_PRUNED); )
1407 }
1408
1409 // Correct quality from max depth to stats
1410 if( !isLastPathElement && node->Quality() == FInfo::Qualities::MAX_DEPTH_REACHED)
1411 node->SetQuality(FInfo::Qualities::STATS);
1412 }
1413
1414 return false;
1415}
1416
1417} // namespace alib::files[::anonymous]
1418
1419#endif // !DOXYGEN
1420
1421// --------------------------------------------------------------------------------------------------
1422//--- ScanFiles()
1423//--------------------------------------------------------------------------------------------------
1424enum FInfo::Qualities ScanFiles( FTree& tree,
1425 ScanParameters& parameters,
1426 std::vector<ResultsPaths>& resultPaths
1427 IF_ALIB_THREADS( , SharedLock* lock) )
1428
1429{
1430 Log_SetDomain( "ALIB/FILES", Scope::Path)
1431 Log_SetDomain( "SCAN" , Scope::Filename)
1432
1433 ALIB_DBG( if( alib::FILES.IsBootstrapped())
1434 {
1435 Log_SetDomain( "ALIB/FILES", Scope::Path)
1436 Log_SetDomain( "SCAN" , Scope::Filename)
1437 } )
1438
1439
1440 //-------------------------------------- get real path ---------------------------------------
1441 Path path(parameters.StartPath);
1442 Path realPath;
1443 realPath.Terminate();
1444
1445 #if ( (defined(__GLIBCXX__) && !defined(__MINGW32__)) \
1446 || defined(__APPLE__) \
1447 || defined(__ANDROID_NDK__) ) && !ALIB_FILES_FORCE_STD_SCANNER
1448 ALIB_STRINGS_TO_NARROW(path , nPath , 512)
1449 ALIB_STRINGS_TO_NARROW(realPath, nRealPath, 512)
1450 if(!realpath(nPath.Terminate(), nRealPath.VBuffer() ) ) switch (errno)
1451 {
1452 case EACCES: ALIB_DBG(errno= 0;) return FInfo::Qualities::NO_ACCESS;
1453 case ENOENT: ALIB_DBG(errno= 0;) return FInfo::Qualities::NOT_EXISTENT;
1454 case ELOOP: ALIB_DBG(errno= 0;) return FInfo::Qualities::CIRCULAR_LINK;
1455 default: ALIB_ERROR("FILES", "Posix raised ({}) {!Q} on resolving start path {!Q}",
1456 errno, SystemErrors(errno), path ) ALIB_DBG(errno= 0;)
1457 return FInfo::Qualities::UNKNOWN_ERROR;
1458 }
1459 nRealPath.DetectLength();
1460 #if ALIB_CHARACTERS_WIDE
1461 realPath.Reset(nRealPath);
1462 #endif
1463#else
1464 {
1465 std::error_code errorCode;
1466 fs::path fsRealPath= fs::canonical(fs::path(std::basic_string_view<PathCharType>(path.Buffer(),
1467 size_t(path.Length()))),
1468 errorCode);
1469 ALIB_DBG(if(errno==EINVAL && !errorCode) errno= 0;) // this happens!, we do not care, but clean up
1470 ALIB_DBG(if(errno==ENOENT && !errorCode) errno= 0;)
1471
1472 ALIB_WARNINGS_ALLOW_SPARSE_ENUM_SWITCH
1473 if(errorCode) switch( SystemErrors(errorCode.value()) )
1474 { // we ignore this: std::fs would not create the "real path" if the final directory is not accessible.
1475 case SystemErrors::eacces: return FInfo::Qualities::NO_ACCESS;
1476 case SystemErrors::enoent: return FInfo::Qualities::NOT_EXISTENT;
1477 case SystemErrors::eloop: return FInfo::Qualities::CIRCULAR_LINK;
1478 default: ALIB_ERROR("FILES", "std::filesystem raised ({}) {!Q} on resolving start path {!Q}",
1479 errorCode.value(), errorCode.message(), path ) ALIB_DBG(errno= 0;)
1480 return FInfo::Qualities::UNKNOWN_ERROR;
1481 }
1482 ALIB_WARNINGS_RESTORE
1483 realPath << fsRealPath.c_str();
1484 }
1485
1486 #endif
1487
1488 Log_Info( "Scanning: P= {}\n"
1489 " RP= {}\n"
1490 " F={} DPre={} DPost={} XFS={} AFS={} Depth={}",
1491 parameters.StartPath, realPath,
1492 parameters.FileFilter .get() ? 'Y':'N',
1493 parameters.DirectoryFilterPreRecursion .get() ? 'Y':'N',
1494 parameters.DirectoryFilterPostRecursion.get() ? 'Y':'N',
1495 parameters.CrossFileSystems ? 'Y':'N', parameters.IncludeArtificialFS ? 'Y':'N',
1496 parameters.MaxDepth == ScanParameters::InfiniteRecursion ? String(A_CHAR("Inf"))
1497 : String128(parameters.MaxDepth)
1498 )
1499
1500 //-------------------------------------- start scanning --------------------------------------
1501 ALIB_DBG( errno=0; )
1502 auto firstResultPos= resultPaths.size();
1503 FInfo::DirectorySums dummySums;
1504
1505 startScan( tree, realPath, parameters, dummySums, resultPaths IF_ALIB_THREADS( , lock) );
1506
1507 Log_Info( "Scan Results: ", resultPaths.size() - firstResultPos )
1508 Log_Prune( int cntPaths= 0;
1509 for( auto& it : resultPaths )
1510 {
1511 Log_Info( " Path {}: {} {} (Q={} D={}/F={}}",
1512 cntPaths++, it.Existed ? ' ' : '+',
1513 it.RealPath,
1514 it.Node->Quality(),
1515 it.Node->Quality() > FInfo::Qualities::STATS && it.Node->IsDirectory() ? it.Node.Value().Sums().CountDirectories() : 0,
1516 it.Node->Quality() > FInfo::Qualities::STATS && it.Node->IsDirectory() ? it.Node.Value().Sums().CountNonDirectories(): 0 )
1517 } )
1518
1519
1520
1521 return (*(resultPaths.begin() + int(firstResultPos))).Node->Quality();
1522}
1523
1524#undef DBG_CHECKERRNO
1525
1526} // namespace [alib::files]
1527
1528#include "alib/lang/callerinfo_methods.hpp"
The entry type which is embedded in each tree node.
Definition finfo.hpp:26
@ STATS
Only stats (size, date, owner, etc.) read.
@ RESOLVED
Read symlink target strings.
@ UNKNOWN_ERROR
Unknown scanner failure.
@ BROKEN_LINK
A symbolic link targets a non-existent file or directory.
@ NO_ACCESS_SL_TARGET
Scanner failure due to limited access rights.
@ NOT_EXISTENT
Set if a given start path does not exist.
int IsAbsolute() const
Definition path.hpp:347
TAString & Append(const TCharSrc *src, integer srcLength)
integer IndexOf(TChar needle, integer startIdx=0) const
Definition string.hpp:896
constexpr integer Length() const
Definition string.hpp:326
TChar CharAtStart() const
Definition string.hpp:466
TChar CharAtEnd() const
Definition string.hpp:488
constexpr const TChar * Buffer() const
Definition string.hpp:319
#define ALIB_WARNING(...)
Definition alib.hpp:1268
#define IF_ALIB_THREADS(...)
Definition alib.hpp:352
#define A_CHAR(STR)
#define Log_IsActive(result,...)
Definition macros.inl:85
#define ALIB_STRING_RESETTER(astring)
#define ALIB_WARNINGS_RESTORE
Definition alib.hpp:849
#define ALIB_STRINGS_TO_NARROW( src, dest, bufSize)
#define Log_Prune(...)
Definition macros.inl:55
#define ALIB_ERROR(...)
Definition alib.hpp:1267
#define ALIB_WARNINGS_ALLOW_SPARSE_ENUM_SWITCH
Definition alib.hpp:785
#define ALIB_ASSERT_ERROR(cond,...)
Definition alib.hpp:1271
#define ALIB_ASSERT_WARNING(cond,...)
Definition alib.hpp:1272
#define ALIB_DBG(...)
Definition alib.hpp:390
#define ALIB_FILES_FORCE_STD_SCANNER
Definition fscanner.hpp:19
#define ALIB_CHARACTERS_WIDE
Definition alib.hpp:434
String DBG_FILES_SCAN_VERBOSE_LOG_FORMAT
This is the reference documentation of sub-namespace system of module ALib BaseCamp.
Definition basecamp.cpp:75
@ Current
The current directory of the process.
Definition alib.cpp:69
files::File File
Type alias in namespace alib.
Definition ftree.hpp:1046
files::FilesCamp FILES
The singleton instance of ALib Camp class FilesCamp.
Definition filescamp.cpp:23
LocalString< 128 > String128
Type alias name for TLocalString<character,128>.
lang::system::SystemErrors SystemErrors
Type alias in namespace alib.
Recursively accumulated values for directories.
Definition finfo.hpp:196
Input parameters to function ScanFiles.
Definition fscanner.hpp:51
SymbolicLinks LinkTreatment
Denotes how symbolic links are treated.
Definition fscanner.hpp:71
unsigned int MaxDepth
The maximum recursion depth. Defaults to InfiniteRecursion.
Definition fscanner.hpp:74
@ DONT_RESOLVE
Demands not to resolve symbolic links in any way.