Skip to Navigation

User login

Username: *

Password: *

• Request new password

• jigdo
• Development
• Debian jigdo mini-HOWTO

Home › jigdo › Development › jigdo API

jigdo API

Last update by Admin on 2010-05-23

• Namespaces
• Class hierarchy
• Class list
• File list
• Namespace Members
• Class Members
• File Members

scan.hh

Go to the documentation of this file.

/* $Id: scan.hh,v 1.33 2003/03/03 20:47:17 richard Exp $ -*- C++ -*-
  __   _
  |_) /|  Copyright (C) 2001-2002  |  richard@
  | \/¯|  Richard Atterer          |  atterer.net
  ¯ '` ¯
  This program is free software; you can redistribute it and/or modify
  it under the terms of the GNU General Public License, version 2. See
  the file COPYING for details.

*/
#ifndef SCAN_HH
#define SCAN_HH

#include <config.h>

#include <list>
#include <set>
#include <vector>
#include <time.h>
#include <sys/stat.h>

#include <cachefile.hh>
#include <debug.hh>
#include <md5sum.hh>
#include <recursedir.fh>
#include <rsyncsum.hh>
#include <scan.fh>
#include <string.hh>
//______________________________________________________________________

class LocationPath {
  friend class JigdoCache;
public:
  bool operator<(const LocationPath& x) const { return path < x.path; }
  const string& getPath() const { return path; }
  const string& getLabel() const { return label; }
  LocationPath& setLabel(const string& l) { label = l; return *this; }
  const string& getUri() const { return uri; }
private:
  template <class StringP, class StringL, class StringU>
  inline LocationPath(const StringP& p, const StringL& l,
                      const StringU& u = "");
  // Using default dtor & copy ctor
  // The data members are accessed directly by JigdoCache methods
  string path; // name of directory
  string label; // short label to identify path in location list
  string uri; // e.g. URL for directory on FTP server
};

typedef set<LocationPath> LocationPathSet;
//______________________________________________________________________

class FilePart {
  friend class JigdoCache;
public:
  inline const string& getPath() const;
  LocationPathSet::iterator getLocation() { return path; }
  inline const string& leafName() const;
  inline uint64 size() const;
  inline time_t mtime() const;
  inline const MD5* getSums(JigdoCache* c, size_t blockNr);
  inline const MD5Sum* getMD5Sum(JigdoCache* c);
  inline const RsyncSum64* getRsyncSum(JigdoCache* c);

  inline void markAsDeleted(JigdoCache* c);
  bool deleted() const { return fileSize == 0; }

  ~FilePart() { }
  // default copy ctor / assignment op
  //__________

private:
  inline FilePart(LocationPathSet::iterator p, string rest, uint64 fSize,
                  time_t fMtime);

  /* Called when the methods getSums/getMD5Sum() need data read from
     file. Might return null. */
  const MD5* getSumsRead(JigdoCache* c, size_t blockNr);
  const MD5Sum* getMD5SumRead(JigdoCache* c);
  //__________

  /* There are 3 states of a FilePart:
     a) sums.empty(): File has not been read from so far
     b) !sums.empty() && !mdValid: sums[0] and rsyncSum are valid
     c) !sums.empty() && mdValid: all sums[] and rsyncSum and md5Sum valid*/

  LocationPathSet::iterator path;
  string pathRest; // further dir names after "path", and leafname of file
  uint64 fileSize;
  time_t fileMtime;

  /* RsyncSum64 of the first MkTemplate::blockLength bytes of the
     file. */
  RsyncSum64 rsyncSum;
  bool rsyncValid() const { return sums.size() > 0; }

  /* File is split up into chunks of length md5BlockLength (the last
     one may be smaller) and the MD5 checksum of each is
     calculated. */
  vector<MD5> sums;

  /* Hash of complete file contents. mdValid is true iff md5Sum is
     cached and valid. NB, it is possible that mdValid==true, but
     sums.size()==0, after md5BlockSize has been changed. If
     sums.size()==fileSize/md5ChunkSize, then not necessarily
     mdValid==true */
  MD5Sum md5Sum;
  enum Flags {
    EMPTY = 0,
    // Bit flag is set iff md5Sum contains the whole file's md5 sum
    MD_VALID = 1,
    /* This file was looked up in the cache file (whether successfully
       or not doesn't matter) - don't look it up again. */
    WAS_LOOKED_UP = 2,
    // Write this file's info into the cache file during ~JigdoCache()
    TO_BE_WRITTEN = 4
  };
  Flags flags;
  bool getFlag(Flags f) const { return (flags & f) != 0; }
  inline void setFlag(Flags f);
  inline void clearFlag(Flags f);
  bool mdValid() const { return getFlag(MD_VALID); }

# if HAVE_LIBDB
  // Offsets for binary representation in database (see cachefile.hh)
  enum {
    BLOCKLEN = 0, MD5BLOCKLEN = 4, MD5BLOCKS = 8, RSYNCSUM = 12,
    FILE_MD5SUM = 20, PART_MD5SUM = 36
  };

  size_t unserializeCacheEntry(const byte* data, size_t dataSize,
      size_t md5BlockLength); // Byte stream => FilePart
  struct SerializeCacheEntry; // FilePart => byte stream
  friend struct SerializeCacheEntry;
# endif
};
//______________________________________________________________________

struct JigdoCacheError : Error {
  explicit JigdoCacheError(const string& m) : Error(m) { }
  explicit JigdoCacheError(const char* m) : Error(m) { }
};

class JigdoCache {
  friend class FilePart;
public:
  class ProgressReporter;
  explicit JigdoCache(const string& cacheFileName,
      size_t expiryInSeconds = 60*60*24*30, size_t bufLen = 128*1024,
      ProgressReporter& pr = noReport);
  ~JigdoCache();

  template <class RecurseDir>
  inline void readFilenames(RecurseDir& rd);
  void setParams(size_t blockLen,size_t md5BlockLen);
  size_t getBlockLen() const { return blockLength; }
  size_t getMD5BlockLen() const { return md5BlockLength; }

  inline void setReadAmount(size_t bytes);
  void deallocBuffer() { buffer.resize(0); }

  ProgressReporter* getReporter() { return &reporter; }

  inline void setCheckFiles(bool check) { checkFiles = check; }
  inline bool getCheckFiles() const { return checkFiles; }

  inline size_t size() const { return nrOfFiles; }

  template <class StringP, class StringL, class StringU>
  const LocationPathSet::iterator addLabel(
    const StringP& path, const StringL& label, const StringU& uri = "");

  typedef FilePart value_type;
  typedef FilePart& reference;
  //____________________

  class iterator {
    friend class JigdoCache;
    friend class FilePart;
  public:
    iterator() { }
    inline iterator& operator++(); // might throw(RecurseError, bad_alloc)
    FilePart& operator*() { return *part; }
    FilePart* operator->() { return &*part; }
    // Won't compare cache members - their being different is usu. a bug
    bool operator==(const iterator& i) const {
      Paranoid(cache == i.cache);
      return part == i.part;
    }
    bool operator!=(const iterator& i) const { return !(*this == i); }
    // Default dtor
  private:
    iterator(JigdoCache* c, const list<FilePart>::iterator& p)
      : cache(c), part(p) { }
    JigdoCache* cache;
    list<FilePart>::iterator part;
  };
  friend class JigdoCache::iterator;
  inline iterator begin();
  iterator end() { return iterator(this, files.end()); }
  //____________________

private:
  // Read one filename from recurseDir and (if success) add entry to "files"
  void addFile(const string& name);
  static ProgressReporter noReport;

  size_t blockLength, md5BlockLength;

  /* Check if files exist in the filesystem */
  bool checkFiles;

  /* List of files in the cache (not vector<> because jigdo-file keeps
     ptrs, and if a vector realloc()s, all elements' addresses may
     change) */
  list<FilePart> files;
  // Equal to files.size() less any files that are deleted()
  size_t nrOfFiles;
  // Temporarily used during readFilenames()
  static struct stat fileInfo;

  // Look up LocationPath by directory name
  LocationPathSet locationPaths;

  size_t readAmount;
  vector<byte> buffer;
  ProgressReporter& reporter;

# if HAVE_LIBDB
  CacheFile* cacheFile;
  size_t cacheExpiry;
# endif
};
//______________________________________________________________________

class JigdoCache::ProgressReporter {
public:
  virtual ~ProgressReporter() { }
  virtual void error(const string& message);
  virtual void info(const string& message);
  virtual void scanningFile(const FilePart* file, uint64 offInFile);
};
//______________________________________________________________________

FilePart::FilePart(LocationPathSet::iterator p, string rest, uint64 fSize,
                   time_t fMtime)
  : path(p), pathRest(rest), fileSize(fSize), fileMtime(fMtime),
    rsyncSum(), sums(), md5Sum(), flags(EMPTY) {
  //pathRest.reserve(0);
}

const string& FilePart::getPath() const {
  Paranoid(!deleted());
  return path->getPath();
}

const string& FilePart::leafName() const {
  Paranoid(!deleted());
  return pathRest;
}

uint64 FilePart::size() const {
  Paranoid(!deleted());
  return fileSize;
}

time_t FilePart::mtime() const {
  Paranoid(!deleted());
  return fileMtime;
}

const MD5* FilePart::getSums(JigdoCache* c, size_t blockNr) {
  Paranoid(!deleted());
  if (mdValid() || (blockNr == 0 && !sums.empty())) return &sums[blockNr];
  else return getSumsRead(c, blockNr);
}

const MD5Sum* FilePart::getMD5Sum(JigdoCache* c) {
  Paranoid(!deleted());
  if (mdValid()) return &md5Sum;
  else return getMD5SumRead(c);
}

const RsyncSum64* FilePart::getRsyncSum(JigdoCache* c) {
  Paranoid(!deleted());
  if (!rsyncValid()) {
    if (getSumsRead(c, 0) == 0) return 0;
  }
  Paranoid(rsyncValid());
  return &rsyncSum;
}

void FilePart::markAsDeleted(JigdoCache* c) {
  fileSize = 0; sums.resize(0); --(c->nrOfFiles);
}

void FilePart::setFlag(Flags f) {
  flags = static_cast<Flags>(static_cast<unsigned>(flags)
                             | static_cast<unsigned>(f));
}

void FilePart::clearFlag(Flags f) {
  flags = static_cast<Flags>(static_cast<unsigned>(flags)
                             & ~static_cast<unsigned>(f));
}
//________________________________________

void JigdoCache::setReadAmount(size_t bytes) {
  if (bytes < 64*1024) bytes = 64*1024;
  readAmount = bytes;
}

template <class RecurseDir>
void JigdoCache::readFilenames(RecurseDir& rd) {
  string name;
  while (true) {
    bool status = rd.getName(name, &fileInfo, checkFiles); // Might throw error
    if (status == FAILURE) return; // No more names
    off_t stSize = fileInfo.st_size;
#   if HAVE_LIBDB
    if (!checkFiles) {
      const byte* data;
      size_t dataSize;
      try {
        if (cacheFile->findName(data, dataSize, name, stSize,
                                fileInfo.st_mtime).failed())
          continue;
      } catch (DbError e) {
        string err = subst(_("Error accessing cache: %1"), e.message);
        reporter.error(err);
      }
    }
#   endif
    if (stSize == 0) continue; // Skip zero-length files
    addFile(name);
  }
}

JigdoCache::iterator JigdoCache::begin() {
  list<FilePart>::iterator i = files.begin();
  while (i != files.end() && i->deleted()) ++i;
  return iterator(this, i);
}

JigdoCache::iterator& JigdoCache::iterator::operator++() {
  list<FilePart>::iterator end = cache->files.end();
  do ++part; while (part != end && part->deleted());
  return *this;
}
//______________________________________________________________________

template <class StringP, class StringL, class StringU>
LocationPath::LocationPath(const StringP& p, const StringL& l,
                           const StringU& u)
  : path(p), label(l), uri(u) {
  //path.reserve(0); label.reserve(0); uri.reserve(0);
}
//________________________________________

/* NB: If an entry with the given path already exists, we must not
   delete it and create another, because it is referenced from at
   least one FilePart. Because of our version of
   LocationPath::operator<, only the path matters during the find(). */
template <class StringP, class StringL, class StringU>
const LocationPathSet::iterator
    JigdoCache::addLabel(const StringP& path, const StringL& label,
                         const StringU& uri) {
  LocationPath tmp(path, label, uri);
  if (!tmp.path.empty() && tmp.path[tmp.path.length() - 1] != DIRSEP)
    tmp.path += DIRSEP;
  LocationPathSet::iterator i = locationPaths.find(tmp);
  if (i == locationPaths.end()) {
    return locationPaths.insert(tmp).first; // Create new entry
  } else {
    implicit_cast<string>(i->label) = tmp.label; // Overwrite old entry
    implicit_cast<string>(i->uri) = tmp.uri;
    return i;
  }
}

#endif

---

• Photography
• Software
• jigdo
• Research
• About

© Copyright 2014 Richard Atterer

 Top of page  •  Home  •  Sitemap  •  Contact  •  About