1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228
|
// --------------------------------------------------------------------------
//
// File
// Name: BackupStoreFile.h
// Purpose: Utils for manipulating files
// Created: 2003/08/28
//
// --------------------------------------------------------------------------
#ifndef BACKUPSTOREFILE__H
#define BACKUPSTOREFILE__H
#include <cstdlib>
#include <memory>
#include <cstdlib>
#include "BackupClientFileAttributes.h"
#include "BackupStoreFilename.h"
#include "IOStream.h"
#include "ReadLoggingStream.h"
#include "RunStatusProvider.h"
typedef struct
{
int64_t mBytesInEncodedFiles;
int64_t mBytesAlreadyOnServer;
int64_t mTotalFileStreamSize;
} BackupStoreFileStats;
// Uncomment to disable backwards compatibility
//#define BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
// Output buffer to EncodeChunk and input data to DecodeChunk must
// have specific alignment, see function comments.
#define BACKUPSTOREFILE_CODING_BLOCKSIZE 16
#define BACKUPSTOREFILE_CODING_OFFSET 15
// Have some memory allocation commands, note closing "Off" at end of file.
#include "MemLeakFindOn.h"
// --------------------------------------------------------------------------
//
// Class
// Name: DiffTimer
// Purpose: Interface for classes that can keep track of diffing time,
// and send SSL keepalive messages
// Created: 2006/01/19
//
// --------------------------------------------------------------------------
class DiffTimer
{
public:
DiffTimer();
virtual ~DiffTimer();
public:
virtual void DoKeepAlive() = 0;
virtual int GetMaximumDiffingTime() = 0;
virtual bool IsManaged() = 0;
};
// --------------------------------------------------------------------------
//
// Class
// Name: BackupStoreFile
// Purpose: Class to hold together utils for manipulating files.
// Created: 2003/08/28
//
// --------------------------------------------------------------------------
class BackupStoreFile
{
public:
class DecodedStream : public IOStream
{
friend class BackupStoreFile;
private:
DecodedStream(IOStream &rEncodedFile, int Timeout);
DecodedStream(const DecodedStream &); // not allowed
DecodedStream &operator=(const DecodedStream &); // not allowed
public:
~DecodedStream();
// Stream functions
virtual int Read(void *pBuffer, int NBytes, int Timeout);
virtual void Write(const void *pBuffer, int NBytes);
virtual bool StreamDataLeft();
virtual bool StreamClosed();
// Accessor functions
const BackupClientFileAttributes &GetAttributes() {return mAttributes;}
const BackupStoreFilename &GetFilename() {return mFilename;}
int64_t GetNumBlocks() {return mNumBlocks;} // primarily for tests
bool IsSymLink();
private:
void Setup(const BackupClientFileAttributes *pAlterativeAttr);
void ReadBlockIndex(bool MagicAlreadyRead);
private:
IOStream &mrEncodedFile;
int mTimeout;
BackupClientFileAttributes mAttributes;
BackupStoreFilename mFilename;
int64_t mNumBlocks;
void *mpBlockIndex;
uint8_t *mpEncodedData;
uint8_t *mpClearData;
int mClearDataSize;
int mCurrentBlock;
int mCurrentBlockClearSize;
int mPositionInCurrentBlock;
uint64_t mEntryIVBase;
#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
bool mIsOldVersion;
#endif
};
// Main interface
static std::auto_ptr<IOStream> EncodeFile(const char *Filename,
int64_t ContainerID, const BackupStoreFilename &rStoreFilename,
int64_t *pModificationTime = 0,
ReadLoggingStream::Logger* pLogger = NULL,
RunStatusProvider* pRunStatusProvider = NULL);
static std::auto_ptr<IOStream> EncodeFileDiff
(
const char *Filename, int64_t ContainerID,
const BackupStoreFilename &rStoreFilename,
int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex,
int Timeout,
DiffTimer *pDiffTimer,
int64_t *pModificationTime = 0,
bool *pIsCompletelyDifferent = 0
);
static bool VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut = 0, int64_t *pContainerIDOut = 0);
static void CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut);
static void CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut);
static void ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent = 0);
static void DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0);
static std::auto_ptr<BackupStoreFile::DecodedStream> DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0);
static bool CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout);
static std::auto_ptr<IOStream> CombineFileIndices(IOStream &rDiff, IOStream &rFrom, bool DiffIsIndexOnly = false, bool FromIsIndexOnly = false);
// Stream manipulation
static std::auto_ptr<IOStream> ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership);
static void MoveStreamPositionToBlockIndex(IOStream &rStream);
// Crypto setup
static void SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength);
#ifndef HAVE_OLD_SSL
static void SetAESKey(const void *pKey, int KeyLength);
#endif
// Allocation of properly aligning chunks for decoding and encoding chunks
inline static void *CodingChunkAlloc(int Size)
{
uint8_t *a = (uint8_t*)malloc((Size) + (BACKUPSTOREFILE_CODING_BLOCKSIZE * 3));
if(a == 0) return 0;
// Align to main block size
ASSERT(sizeof(unsigned long) >= sizeof(void*)); // make sure casting the right pointer size
uint8_t adjustment = BACKUPSTOREFILE_CODING_BLOCKSIZE
- (uint8_t)(((unsigned long)a) % BACKUPSTOREFILE_CODING_BLOCKSIZE);
uint8_t *b = (a + adjustment);
// Store adjustment
*b = adjustment;
// Return offset
return b + BACKUPSTOREFILE_CODING_OFFSET;
}
inline static void CodingChunkFree(void *Block)
{
// Check alignment is as expected
ASSERT(sizeof(unsigned long) >= sizeof(void*)); // make sure casting the right pointer size
ASSERT((uint8_t)(((unsigned long)Block) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET);
uint8_t *a = (uint8_t*)Block;
a -= BACKUPSTOREFILE_CODING_OFFSET;
// Adjust downwards...
a -= *a;
free(a);
}
static void DiffTimerExpired();
// Building blocks
class EncodingBuffer
{
public:
EncodingBuffer();
~EncodingBuffer();
private:
// No copying
EncodingBuffer(const EncodingBuffer &);
EncodingBuffer &operator=(const EncodingBuffer &);
public:
void Allocate(int Size);
void Reallocate(int NewSize);
uint8_t *mpBuffer;
int mBufferSize;
};
static int MaxBlockSizeForChunkSize(int ChunkSize);
static int EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput);
// Caller should know how big the output size is, but also allocate a bit more memory to cover various
// overheads allowed for in checks
static inline int OutputBufferSizeForKnownOutputSize(int KnownChunkSize)
{
// Plenty big enough
return KnownChunkSize + 256;
}
static int DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize);
// Statisitics, not designed to be completely reliable
static void ResetStats();
static BackupStoreFileStats msStats;
// For debug
#ifndef BOX_RELEASE_BUILD
static bool TraceDetailsOfDiffProcess;
#endif
// For decoding encoded files
static void DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFile);
};
#include "MemLeakFindOff.h"
#endif // BACKUPSTOREFILE__H
|