-
Notifications
You must be signed in to change notification settings - Fork 0
/
cleaning.py
145 lines (124 loc) · 5.11 KB
/
cleaning.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
import subprocess
import time
import os
import glob
import traceback
# combine file downloaded with GNU awk
def combine_list(src_path,output_path):
system = subprocess.Popen([f"gawk '!seen[$0]++' {src_path} > {output_path}"],
# stdout=subprocess.PIPE,
# stderr=subprocess.PIPE,
universal_newlines=True, shell=True)
return(system.communicate())
# remove commented line with sed
def clean_comments(src_path):
system = subprocess.Popen([f"sed -i '/^[[:blank:]]*#/d;s/#.*//' {src_path}"],
# stdout=subprocess.PIPE,
# stderr=subprocess.PIPE,
universal_newlines=True, shell=True)
return(system.communicate())
# sort the file with linux command
def sort_file(src_path,out_path):
system = subprocess.Popen([f"sort {src_path} > {out_path}"],
# stdout=subprocess.PIPE,
# stderr=subprocess.PIPE,
universal_newlines=True, shell=True)
return(system.communicate())
def remove_localhost(line):
remove_list = set()
remove_list.add("localhost")
remove_list.add("localhost.localdomain")
remove_list.add("local")
remove_list.add("broadcasthost")
remove_list.add("ip6-localhost")
remove_list.add("ip6-loopback")
remove_list.add("ip6-localnet")
remove_list.add("ip6-mcastprefix")
remove_list.add("ip6-allnodes")
remove_list.add("ip6-allrouters")
remove_list.add("ip6-allhosts")
remove_list.add("0.0.0.0")
sentences = line.split() # split the line by whitespace
if len(sentences) > 0: # remove empty line
line = sentences[-1] # only get the {domain} part of the line
line = line.lower() # domain is case insensitive so lower it just incase.
if not line in remove_list:
return line
def clean_non_domain_data(path_file):
temp_file = f"{path_file}.tmp"
with open(temp_file,'w', encoding="utf-8") as temp:
with open(path_file, 'r', encoding="utf-8") as current_file: #ISO-8859-1
for raw_line in current_file:
# call remove_localhost to split the line and get only domain
line = remove_localhost(raw_line)
if line is not None:
# print(line)
temp.write(f"{line}\n")
os.replace(temp_file, path_file)
def snooze_for(how_much):
start = time.time()
time.sleep(how_much)
return (time.time() - start)
def main():
# init
path_dwl = os.path.join(f"{os.getcwd()}","dwl")
text_files = [f for f in os.listdir(path_dwl) if f.endswith('.txt')]
gawk_src = os.path.join(f"{os.getcwd()}","dwl","*.txt")
gawk_output = os.path.join(f"{os.getcwd()}","blocklist","unordered.txt")
sort_output = os.path.join(f"{os.getcwd()}","blocklist","ordered.txt")
# loop through downloaded files for removing comments
print("[Processing: Cleaning Comments]")
for i in range(len(text_files)):
i+=1
filename = f"{i}.txt"
src_path = f"dwl/{filename}"
start = time.time()
clean_comments(src_path)
print(f" Clean Comments took: \t{(time.time() - start):.2f} seconds")
print(f"\nSleep for {snooze_for(2):.1f} sec \n")
# loop through downloaded files for removing non domain
print("[Processing: Cleaning Non Domain]")
for i in range(len(text_files)):
i+=1
filename = f"{i}.txt"
src_path = f"dwl/{filename}"
start = time.time()
clean_non_domain_data(src_path)
print(f" Domain Formatting took:\t{(time.time() - start):.2f} seconds")
print(f"\nSleep for {snooze_for(2):.1f} sec \n")
# remove duplicate and combine to one file
print("[Processing: Compiling]")
start = time.time()
combine_list(gawk_src, gawk_output)
print(f" Process took: \t\t{(time.time() - start):.2f} seconds")
print(f"\nSleep for {snooze_for(2):.1f} sec \n")
# sort the output file
print("[Processing: Sorting]")
start = time.time()
sort_file(gawk_output, sort_output)
print(f" Process took: \t\t{(time.time() - start):.2f} seconds")
# remove unordered output
os.remove(gawk_output)
for file in glob.glob(os.path.join(path_dwl, "*.txt")):
os.remove(file)
if __name__ == "__main__":
# try:
# # <<<=================== EXECUTE MAIN ===================>>>
# main()
# except IOError:
# print('Error: Input / Output Error')
# except ValueError:
# print('Error: Value Error')
# except ImportError:
# print('Error: NO module found.')
# except EOFError:
# print('Error: Why did you do an EOF on me?')
# except KeyboardInterrupt:
# print('Error: You are cancelled LOL')
# except:
# print('An error occured, and I don\'t know why')
try:
main()
except Exception as e:
print(e)
traceback.print_exc()