1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
|
#!/usr/bin/env ruby
require 'csv'
require 'json'
require 'async/http/internet'
class RateLimitingError < StandardError; end
@internet = Async::HTTP::Internet.new
@user = ENV['GITHUB_USER']
@token = ENV['GITHUB_TOKEN']
unless @user && @token
fail "export GITHUB_USER and GITHUB_TOKEN!"
end
GITHUB_HEADERS = {
'user-agent' => 'fetch-github-licenses',
'authorization' => Protocol::HTTP::Header::Authorization.basic(@user, @token)
}
RUBYGEMS_HEADERS = {
'user-agent' => 'fetch-github-licenses'
}
def fetch_github_license(homepage_uri)
%r{github.com/(?<owner>.+?)/(?<repo>.+)} =~ homepage_uri
return nil unless repo
response = @internet.get("https://api.github.com/repos/#{owner}/#{repo}/license", GITHUB_HEADERS)
case response.status
when 200
return JSON.parse(response.read).dig('license', 'spdx_id')
when 404
return nil
else
raise response.read
end
ensure
response.finish
end
def fetch_rubygem_license(name, version)
response = @internet.get("https://rubygems.org/api/v2/rubygems/#{name}/versions/#{version}.json", RUBYGEMS_HEADERS)
case response.status
when 200
body = JSON.parse(response.read)
[name, body.dig('licenses', 0) || fetch_github_license(body['homepage_uri'])]
when 404
[name, nil] # from a non rubygems remote
when 429
raise RateLimitingError
else
raise response.read
end
rescue RateLimitingError
response.finish
Console.logger.warn(name) {"Rate limited..."}
Async::Task.current.sleep(1.0)
retry
ensure
response.finish
end
Sync do |parent|
output = CSV.new($stdout)
tasks = ARGF.map do |line|
if line == "GEM\n" .. line.chomp.empty?
/\A\s{4}(?<name>[a-z].+?) \((?<version>.+)\)\n\z/ =~ line
parent.async do
fetch_rubygem_license(name, version)
end if name
end
end.compact
tasks.each do |task|
output << task.wait
end
@internet.instance_variable_get(:@clients).each do |name, client|
puts client.pool
end
end
|
