1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
|
require "bundler"
require "rdoc/task"
require "rake/testtask"
task default: :test
Bundler::GemHelper.install_tasks
RDoc::Task.new do |task|
task.rdoc_dir = "doc"
task.title = "BufferedTokenizer"
task.rdoc_files.include("lib/**/*.rb")
end
Rake::TestTask.new :test do |t|
t.libs << "lib"
t.test_files = FileList["test/**/*.rb"]
end
desc "Benchmark the current implementation"
task :bench do
require "benchmark"
require File.expand_path("lib/buftok", File.dirname(__FILE__))
n = 50000
delimiter = "\n\n"
frequency1 = 1000
puts "generating #{n} strings, with #{delimiter.inspect} every #{frequency1} strings..."
data1 = (0...n).map do |i|
((i % frequency1 == 1 ? "\n" : "") +
("s" * i) +
(i % frequency1 == 0 ? "\n" : "")).freeze
end
frequency2 = 10
puts "generating #{n} strings, with #{delimiter.inspect} every #{frequency2} strings..."
data2 = (0...n).map do |i|
((i % frequency2 == 1 ? "\n" : "") +
("s" * i) +
(i % frequency2 == 0 ? "\n" : "")).freeze
end
Benchmark.bmbm do |x|
x.report("1 char, freq: #{frequency1}") do
bt1 = BufferedTokenizer.new
n.times { |i| bt1.extract(data1[i]) }
end
x.report("2 char, freq: #{frequency1}") do
bt2 = BufferedTokenizer.new(delimiter)
n.times { |i| bt2.extract(data1[i]) }
end
x.report("1 char, freq: #{frequency2}") do
bt3 = BufferedTokenizer.new
n.times { |i| bt3.extract(data2[i]) }
end
x.report("2 char, freq: #{frequency2}") do
bt4 = BufferedTokenizer.new(delimiter)
n.times { |i| bt4.extract(data2[i]) }
end
end
end
|
