File: fileStructure.py

package info (click to toggle)
subuser 0.6.2-3.1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 4,216 kB
  • sloc: python: 5,204; sh: 380; makefile: 73; javascript: 43
file content (351 lines) | stat: -rwxr-xr-x 10,594 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
# -*- coding: utf-8 -*-
# pylint: disable=no-init,old-style-class

"""
A file structure object is a read only view of a file structure, such as a real directory or a git tree.
"""

#external imports
import abc
import hashlib
import os
import stat
import sys
#internal imports
import subuserlib.test
import subuserlib.print
if subuserlib.test.testing:
  hashtestDir = subuserlib.test.hashtestDir

class FileStructure():
  __metaclass__ = abc.ABCMeta

  def ls(self, subfolder,objectType=None):
    """
    Returns a list of file and folder paths.
    Paths are relative to the root of the FileStructure.
    """
    self.assertLegalPath(subfolder)
    return self._ls(subfolder,objectType=objectType)

  @abc.abstractmethod
  def _ls(self,subfolder,objectType=None):
    pass

  def lsFiles(self,subfolder):
    """
    Returns a list of paths to files in the subfolder.
    Paths are relative to the root of the FileStructure.
    """
    self.assertLegalPath(subfolder)
    return self._lsFiles(subfolder)

  @abc.abstractmethod
  def _lsFiles(self,subfolder):
    pass

  def lsFolders(self,subfolder):
    """
    Returns a list of paths to folders in the subfolder.
    Paths are relative to the root of the FileStructure.
    """
    self.assertLegalPath(subfolder)
    return self._lsFolders(subfolder)

  @abc.abstractmethod
  def _lsFolders(self,subfolder):
    pass

  def exists(self,path):
    self.assertLegalPath(path)
    return self._exists(path)

  @abc.abstractmethod
  def _exists(self,path):
    pass

  def read(self,path):
    """
    Returns the contents of the given file.
    """
    self.assertLegalPath(path)
    return self._read(path)

  @abc.abstractmethod
  def _read(self,path):
    pass

  def readBinary(self,path):
    """
    Returns the contents of the given file.
    """
    self.assertLegalPath(path)
    return self._readBinary(path)

  @abc.abstractmethod
  def _readBinary(self,path):
    pass

  @abc.abstractmethod
  def getMode(self,path):
   self.assertLegalPath(path)
   return self._getMode(path)

  def getSize(self,path):
    self.assertLegalPath(path)
    return self._getSize(path)

  @abc.abstractmethod
  def _getSize(self,path):
    pass

  @abc.abstractmethod
  def isLegalSymlink(self,path):
    """
    If the path is a symlink. Make sure it doesn't point out of the structure.
    """
    pass

  def assertLegalPath(self,path):
    """
    Throw an exception if path is a relative path going up outside of the file structure.

    >>> from subuserlib.classes.fileStructure import FileStructure
    >>> fileStructure = BasicFileStructure(subuserlib.classes.fileStructure.hashtestDir)

    These should work.

    >>> fileStructure.assertLegalPath("./foo")
    >>> fileStructure.assertLegalPath("./bar/../foo")

    These shouldn't.

    >>> fileStructure.assertLegalPath("../foo")
    Traceback (most recent call last):
    ...
    OSError: ../foo does not exist in file structure.
    >>> fileStructure.assertLegalPath("./../foo")
    Traceback (most recent call last):
    ...
    OSError: ./../foo does not exist in file structure.
    >>> fileStructure.assertLegalPath("./bar/../../foo")
    Traceback (most recent call last):
    ...
    OSError: ./bar/../../foo does not exist in file structure.
    >>> fileStructure.assertLegalPath("./bar/../../../foo")
    Traceback (most recent call last):
    ...
    OSError: ./bar/../../../foo does not exist in file structure.

    Absolute paths are absolute nonos

    >>> fileStructure.assertLegalPath("/bar")
    Traceback (most recent call last):
    ...
    OSError: /bar does not exist in file structure.

    Checks for symlinks too.
    """
    if path.startswith("/") or os.path.relpath(path,"./").startswith(".."):
      raise IOError(path + " does not exist in file structure.")
    try:
      path = self.realpath(path)
      if not self.isLegalSymlink(path):
        raise IOError(path + " is a symlink which points outside of the filestructure which is not allowed.")
    except AttributeError:
      pass

  def getModeString(self,path):
    """
    Return the human readable mode string for the mode in octal notation.
    """
    self.assertLegalPath(path)
    octalMode = oct(self.getMode(path))
    if sys.version_info[0] == 2:
      octalMode = octalMode[1:]
    if sys.version_info[0] == 3:
      octalMode = octalMode[2:]
    return octalMode

  def hash(self,path,printDebugOutput=False):
    """
    Return the SHA512 hash of the file or directory.

    In the case of directectories, hashes alphabetically and hashes subdirectories first.

    Hashes the following:
      - Relative file path
      - File size as UTF-8 string
      - File contents

    NOTE: File mode is not included in hash because git doesn't actually support meaningful file modes. :(

    Return the hash as a hexidecimal string.

    >>> from subuserlib.classes.fileStructure import FileStructure
    >>> fileStructure = BasicFileStructure(subuserlib.classes.fileStructure.hashtestDir)
    >>> fileStructure.hash("./")
    'b0cd63dd96b76d7a9c61e434b43f0eea408c2dd14dca1f436be0a56bf1f91aa75f4406b9fe9fb2025b84e3445f747a2680d56ca92f5b4fc28a98d8f70586cf15'
    """
    self.assertLegalPath(path)
    hashFunction = hashlib.sha512
    hash = hashFunction()
    # TODO - what about symlinks?
    # TODO - what about devices?
    # TODO - what about sockets?
    # TODO - what about named pipes?
    def hashFile(path):
      file_metadata_string = str(len(path))+" "+path+" "+str(self.getSize(path))+" "
      hash.update(file_metadata_string.encode("utf-8"))
      hash.update(self.readBinary(path))
      hash.update("\n".encode("utf-8"))
      if printDebugOutput:
        subuserlib.print.printWithoutCrashing(file_metadata_string.encode("utf-8").decode("utf-8")+self.readBinary(path).decode("utf-8","replace"))
    def hashDir(path):
      # Hash subdirectories
      subdirs = self.lsFolders(path)
      subdirs.sort()
      for subdir in subdirs:
        if subdir != ".git":
          hashDir(os.path.join(path,subdir))
      # Hash files
      files = self.lsFiles(path)
      files.sort()
      for fileToHash in files:
        hashFile(os.path.join(path,fileToHash))
    hashDir(path)
    return hash.hexdigest()

class BasicFileStructure(FileStructure):
  """
  A FileStructure backed by real files.
  """
  def __init__(self,path):
    self.path = path
    if not os.path.exists(path):
      raise FileNotFoundError(path+" does not exist.")

  def getPathInStructure(self,path):
    """
    Given a relative path within the file structure, return an absolute path.
    """
    return os.path.join(self.path,path)

  def _ls(self, subfolder,objectType=None):
    """
    >>> from subuserlib.classes.fileStructure import FileStructure
    >>> fileStructure = BasicFileStructure(subuserlib.classes.fileStructure.hashtestDir)
    >>> print(",".join(fileStructure.ls("./")))
    bar,blah
    """
    assert(objectType==None)
    paths = []
    path = self.getPathInStructure(subfolder)
    for path in os.listdir(path):
      paths.append(path)
    paths.sort()
    return paths

  def _lsFiles(self,subfolder):
    """
    >>> from subuserlib.classes.fileStructure import FileStructure
    >>> import os
    >>> fileStructure = BasicFileStructure(subuserlib.classes.fileStructure.hashtestDir)
    >>> print(",".join(fileStructure.lsFiles("./")))
    blah
    >>> print(",".join(fileStructure.lsFiles("bar")))
    New York,abacus
    >>> print(",".join(fileStructure.lsFiles("./bar")))
    New York,abacus
    """
    files = []
    for path in self.ls(subfolder):
      if os.path.isfile(self.getPathInStructure(os.path.join(subfolder,path))):
        files.append(path)
    return files

  def _lsFolders(self,subfolder):
    """
    >>> from subuserlib.classes.fileStructure import FileStructure
    >>> import os
    >>> fileStructure = BasicFileStructure(subuserlib.classes.fileStructure.hashtestDir)
    >>> print(",".join(fileStructure.lsFolders("./")))
    bar
    """
    folders = []
    for path in self.ls(subfolder):
      pathInStructure = self.getPathInStructure(os.path.join(subfolder,path))
      if os.path.isdir(pathInStructure):
        folders.append(path)
    return folders

  def _exists(self,path):
    """
    >>> from subuserlib.classes.fileStructure import FileStructure
    >>> import os
    >>> fileStructure = BasicFileStructure(subuserlib.classes.fileStructure.hashtestDir)
    >>> fileStructure.exists("./blah")
    True
    >>> fileStructure.exists("./non-existant")
    False
    """
    return os.path.exists(self.getPathInStructure(path))

  def _read(self,path):
    """
    Reads in a file as a utf-8 string.

    >>> from subuserlib.classes.fileStructure import BasicFileStructure
    >>> import os
    >>> fileStructure = BasicFileStructure(subuserlib.classes.fileStructure.hashtestDir)
    >>> print(fileStructure.read("./blah"))
    blahblah
    <BLANKLINE>
    
    You can only read in files that are actually inside the file structure.
    >>> fileStructure1 = BasicFileStructure(os.path.join(subuserlib.classes.fileStructure.hashtestDir,"bar"))
    >>> print(fileStructure1.read("../blah"))
    Traceback (most recent call last):
    ...
    OSError: ../blah does not exist in file structure.
    """
    with open(self.getPathInStructure(path),"r",encoding="utf-8") as fd:
      return fd.read()

  def _readBinary(self,path):
    """
    >>> from subuserlib.classes.fileStructure import FileStructure
    >>> import os
    >>> fileStructure = BasicFileStructure(subuserlib.classes.fileStructure.hashtestDir)
    >>> print(fileStructure.read("./blah"))
    blahblah
    <BLANKLINE>
    """
    with open(self.getPathInStructure(path),"rb") as fd:
      return fd.read()

  def _getMode(self,path):
    """
    >>> from subuserlib.classes.fileStructure import FileStructure
    >>> import os
    >>> fileStructure = BasicFileStructure(subuserlib.classes.fileStructure.hashtestDir)
    >>> print(fileStructure.getModeString("./blah"))
    100664
    """
    return os.stat(self.getPathInStructure(path))[stat.ST_MODE]

  def _getSize(self,path):
    """
    >>> from subuserlib.classes.fileStructure import FileStructure
    >>> import os
    >>> fileStructure = BasicFileStructure(subuserlib.classes.fileStructure.hashtestDir)
    >>> print(fileStructure.getSize("./blah"))
    9
    """
    return os.stat(self.getPathInStructure(path))[stat.ST_SIZE]

  def realpath(self,path):
    return os.path.realpath(self.getPathInStructure(path))

  def isLegalSymlink(self,path):
    return not os.path.relpath(path,self.path).startswith("..")