#! /usr/bin/env python3.3 # Copyright 2014 Claudio Moretti # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as # published by the Free Software Foundation, either version 3 of the # License, or (at your option) any later version. # # This little piece of software works by downloading the Alexa Top 1M website list, which freely available, # then it uses `git diff` to generate a list of XML ruleset files that are in the master branch but not in stable. # Finally, it compares the two and prints the file name and path of every ruleset file that # a) is in master but not in stable and # b) has a target in the Alexa Top1M list # import sys import csv import xml.etree.ElementTree as etree import subprocess import random import urllib.request import urllib.error import zipfile import os import time # Variables and constants sitesList = [] # Temporary file containing the `git diff` between master and stable tmpRulesFileName = "/tmp/rulesDiff-" + format(random.randrange(1,65535)) # Feel free to enlarge if needed # URL of the Alexa Top1M alexaTop1MURL = "http://s3.amazonaws.com/alexa-static/top-1m.csv.zip" # alexaTop1MURL = "http://127.0.0.1/top-1m.csv.zip" # Temporary file name, to aboid conflicts tmpAlexaFileName = "/tmp/alexa-top1M-" + format(random.randrange(1,65535)) + ".csv" # Logfile. Records the same output as the script logFileName = "/tmp/alexa-ruleset-log-" + format(random.randrange(1,65535)) + ".log" # Filename of the CSV file contained in the Alexa zipfile tmpAlexaZipFileContents = 'top-1m.csv' # Absolute path of the git repo (the folder containing src/) # Remember to change this accordingly to your system, if you ever move the script # # By default, it refers to the parent directory of the one containing the script # because the script was put in utils/ # # __NEEDS A TRAILING SLASH__ # # gitRepositoryPath = os.path.abspath(os.path.join(os.curdir, os.pardir)) gitRepositoryPath = os.path.abspath(os.path.join(os.curdir, os.pardir)) + "/" # Maximum number of websites to use in the Alexa Top 1M (i.e. it's no longer 1M but maxSitesNumber) # Set to -1 for 'unlimited' maxSitesNumber = 1000 # Functions def ruleLookup(target): try: # list.index(value) throus an exception for a "not found", so if it throws it, it's not found sitesList.index(target) return 1 except: return 0 # Fetch the Alexa Top 1M - http://stackoverflow.com/questions/1517616/stream-large-binary-files-with-urllib2-to-file try: print("Retrieving Alexa Top1M from", alexaTop1MURL) tmpAlexaZipFileName, headers = urllib.request.urlretrieve(alexaTop1MURL) print("File downloaded and stored in %s" % tmpAlexaZipFileName) except urllib.error.URLError as e: print("Failed to download Alexa Top 1M") sys.exit('Error message: %s' % e) # Now unzip it try: # Extract in /tmp/ print("Start extracting %s" % tmpAlexaZipFileName) tmpAlexaZipFile = zipfile.ZipFile(tmpAlexaZipFileName,'r') tmpAlexaZipFile.extractall('/tmp/') except zipfile.BadZipfile: sys.exit("The zip file %s is corrupted.",tmpAlexaZipFileName) try: # Rename the file to match the file with the random in it os.rename('/tmp/' + tmpAlexaZipFileContents,tmpAlexaFileName) print("Alexa Top1M retrieved and stored in %s" % tmpAlexaFileName) except OSError as e: print("Failed to rename /tmp/top-1M.csv to %s." % (tmpAlexaFileName)) sys.exit('Error message: %s' % (e)) # Handles reading the Alexa Top 1M and pushing all sites in a list sitesReader = csv.reader(open(tmpAlexaFileName), delimiter=',', quotechar='"') for row in sitesReader: try: # Since some Alexa sites are not FQDNs, split where there's a "/" and keep ony the first part siteFQDN = sitesList.append(row[1].split("/",1)[0]) # print("Line %s: %s" % (sitesReader.line_num, sitesList[len(sitesList) - 1])) # Outputs the current line if sitesReader.line_num == maxSitesNumber: break except csv.Error as e: sys.exit('file %s, line %d: %s' % (tmpAlexaFileName, sitesReader.line_num, e)) # `git diff` the master revision against stable, rules folder only try: print("Create git diff between master and stable in %s" % tmpRulesFileName) tmpRulesFile = open(tmpRulesFileName,"w") #subprocess.call(['git', 'diff', '--name-status', 'master..remotes/origin/stable', '../src/chrome/content/rules'], stdout=tmpRulesFile) subprocess.call(['git', 'diff', '--name-status', 'remotes/origin/stable..master', '../src/chrome/content/rules'], stdout=tmpRulesFile) tmpRulesFile.close() except OSError as e: sys.exit('An OSError exception was raised: %s' % (e)) rulesList = open(tmpRulesFileName, 'r') logFile = open(logFileName,'w') logFile.write("Log file generated on %s.\nPaths are relative to the root directory of the git repo.\n\n" % time.strftime("%Y-%m-%d %H:%M:%S")) # Let's keep track of how many rules were added and how many were modified # Must be declared here or won't be available at the end of the loop countAddedRules = 0 countEditedRules = 0 # Start parsing the list for line in rulesList: try: # Split into "file mode in commit + file path" ruleFile = line.split() found = 0 # If file mode is "A" (add) or "M" (edited) if ruleFile[0] == "A" or ruleFile[0] == "M": # If file was added or edited between stable and master, parse ruleFileObject= open(gitRepositoryPath + ruleFile[1]) ruleText = etree.parse(ruleFileObject) # ADJUST FILE PATH (here is '../') IF YOU MOVE THE SCRIPT - XXX: Obsolete warning? for target in ruleText.findall('target'): FQDN = target.get('host') # URL of the website if ruleLookup(FQDN) == 1: # Look it up in the sitesList # Message different according to file mode if ruleFile[0] == "A": # New found = "NEW" countAddedRules = countAddedRules + 1 break elif ruleFile[0] == "M": # Edited found = "EDITED" countEditedRules = countEditedRules + 1 break # If found, print it TABULATED if found != 0: print("%s:\t%s" % (found, ruleFile[1])) logFile.write("%s:\t%s" % (found, ruleFile[1])) # else ignore # There are some problems with file name encoding. So, for now, just print an error and pass except FileNotFoundError as e: # Won't happen before line.split() is invoked print("File not found:", ruleFile[1]) # logFile.write ("File not found: %s\n" % ruleFile[1]) logFile.write("%s\n" % e) pass except IOError as ioe: #Treated same as FileNotFoundError print("File not found:", ruleFile[1]) # logFile.write ("File not found: %s\n" % ruleFile[1]) logFile.write("%s\n" % e) pass # Print our simple statistics print("\n\nStatistics:\nParsed rules: %s\nNewly added rules: %s\nEdited rules: %d" % (maxSitesNumber, countAddedRules, countEditedRules)) logFile.write("\n\nStatistics:\nParsed rules: %s\nNewly added rules: %s\nEdited rules: %d" % (maxSitesNumber, countAddedRules, countEditedRules)) print("\n\nLog file can be found at %s" % logFileName) # Close the rules file rulesList.close() # And the log file logFile.close()