com pftt2: Automated Reporting supporting functions (snapshot grabbing, etc...): PFTT/lib/report/comparison/by_platform/func_build _comparison.rb
PFTT/lib/report/comparison/by_platform/f unc_release_comparison.rb PFTT/lib/report/comparison/by _platform/perf_build_comparison.rb PFTT/lib/report/co
mparison/by_platform/perf_release_comparison.rb PFTT/ lib/server/fbcd_mailer.rb PFTT/lib/server/lock_server .rb PFTT/lib/server/network_monitor.rb
PFTT/lib/ser
| From: | Matt Ficken | Date: | Sun, 23 Oct 2011 07:53:54 +0000 |
| Subject: | com pftt2: Automated Reporting supporting functions (snapshot grabbing, etc...): PFTT/lib/report/comparison/by_platform/func_build _comparison.rb PFTT/lib/report/comparison/by_platform/f unc_release_comparison.rb PFTT/lib/report/comparison/by _platform/perf_build_comparison.rb PFTT/lib/report/co mparison/by_platform/perf_release_comparison.rb PFTT/ lib/server/fbcd_mailer.rb PFTT/lib/server/lock_server .rb PFTT/lib/server/network_monitor.rb PFTT/lib/ser |
||
| Groups: | php.qa | ||
| Request: | Send a blank email to php-qa+get-67161@lists.php.net to get a copy of this message | ||
Commit: f87834959e87e7d4c8f4f13e76a782c2caa457db
Author: Matt Ficken <v-mafick@.redmond.corp.microsoft.com> Sun, 23 Oct 2011
00:53:54 -0700
Parents: 6749585dbfe6ebb38d8638eadd771dbef730aab0
Branches: master
Link: http://git.php.net/?p=pftt2.git;a=commitdiff;h=f87834959e87e7d4c8f4f13e76a782c2caa457db
Log:
Automated Reporting supporting functions (snapshot grabbing, etc...)
Changed paths:
A PFTT/lib/report/comparison/by_platform/func_build_comparison.rb
D PFTT/lib/report/comparison/by_platform/func_release_comparison.rb
A PFTT/lib/report/comparison/by_platform/perf_build_comparison.rb
D PFTT/lib/report/comparison/by_platform/perf_release_comparison.rb
A PFTT/lib/server/fbcd_mailer.rb
A PFTT/lib/server/lock_server.rb
A PFTT/lib/server/network_monitor.rb
A PFTT/lib/server/psb.rb
A PFTT/lib/server/snapshot_getter.rb
A PFTT/lib/server/svn_updater.rb
diff --git a/PFTT/lib/report/comparison/by_platform/func_build_comparison.rb b/PFTT/lib/report/comparison/by_platform/func_build_comparison.rb new file mode 100644 index 0000000..2501647 --- /dev/null +++ b/PFTT/lib/report/comparison/by_platform/func_build_comparison.rb @@ -0,0 +1,206 @@ + +module Report + module Comparison + module ByPlatform + class FuncBuildComparison < Base + + def title + 'Functional Build Comparison(FBC) Report' + end + + def write_intro + # TODO time, title + "<html><body><h1>#{title}</h1><p>Comparing @resultset_a.title (Base) with @resultset_b.title (Test) (using tests current as of @resultset_a.test_time</p><p>Click <a href=\"#{resultsets_comparison_url}\" target=\"_blank\">here</a> for a customizable comparison</p>" + end + + def write_end + "<p>* - tests missing required sections (test is not runnable)</p><p>** - tests with unsupported sections (test is not runnable)</p></body></html>" + end + + def write_table(cm) + # colors to use + bgcolor_os = '#f2f2f2' + bgcolor_test = '#ccff66' + bgcolor_skip = '#ffff00' + bgcolor_test_bug = '#ffc000' + bgcolor_pftt_bug = '#f28020' + bgcolor_telemetry = '#ccff66' + bgcolor_good = '#96dc28' + bgcolor_bad = '#ff0000' + # + + [['context_1', 'context_2'], ['context_1', 'context_3']].each{|ctx_set| + + ['IIS with WinCache', 'IIS', 'Command Line', 'Apache with APC', 'Apache with APC and IGBinary', 'Apache'].each{|mw_txt| + cm.add_row({:text=>(cm.html?)?'<strong>'+mw_txt+'</strong>':mw_txt, :colspan=>22, :center=>true}) + cm.add_row( + {:text=>'OS', :colspan=>2, :bgcolor=>bgcolor_os}, + {:text=>'', :colspan=>8, :bgcolor=>bgcolor_test}, + {:text=>'Skip', :colspan=>4, :bgcolor=>bgcolor_skip, :center=>true}, + {:text=>'Test Bug', :colspan=>4, :bgcolor=>bgcolor_test_bug, :center=>true}, + {:text=>'PFTT Bug', :colspan=>2, :bgcolor=>bgcolor_pftt_bug, :center=>true}, + {:text=>'Telemetry', :colspan=>2, :bgcolor=>bgcolor_telemetry, :center=>true} + ) + cm.add_row( + {:text=>'', :bgcolor=>bgcolor_os, :colspan=>2}, + {:text=>'Pass Rate', :bgcolor=>bgcolor_test, :colspan=>2, :center=>true}, + {:text=>'Pass', :bgcolor=>bgcolor_test, :colspan=>2, :center=>true}, + {:text=>'Fail', :bgcolor=>bgcolor_test, :colspan=>2, :center=>true}, + {:text=>'XFail(Pass)', :bgcolor=>bgcolor_test, :colspan=>2, :center=>true}, + {:text=>'Skip', :bgcolor=>bgcolor_skip, :colspan=>2, :center=>true}, + {:text=>'XSkip', :bgcolor=>bgcolor_skip, :colspan=>2, :center=>true}, + {:text=>'XFail(Work)', :bgcolor=>bgcolor_test_bug, :colspan=>2, :center=>true}, + {:text=>'Bork*', :bgcolor=>bgcolor_test_bug, :colspan=>2, :center=>true}, + {:text=>'Unsupported**', :bgcolor=>bgcolor_pftt_bug, :colspan=>2, :center=>true}, + {:text=>'', :bgcolor=>bgcolor_telemetry, :colspan=>2} + ) + + results = ['A Result'] + if results.empty? + cm.add_row({:text=>'No Results', :colspan=>22, :center=>true}) + else + results.each{|result| + platform = 'Win 2003r2 x86 SP0, SP1' + pass_rate = 50 + change_pass_rate = -5 + pass = 5000 + change_pass = -100 + fail = 5000 + change_fail = +50 + xfail_pass = 0 + change_xfail_pass = -10 + skip = 0 + change_skip = -100 + xskip = 0 + change_xskip = -100 + xfail_work = 0 + change_xfail_work = -100 + bork = 0 + change_bork = 0 + unsupported = 0 + change_unsupported = +10 + telemetry_url_base = 'file://///10.200.51.33/share/PFTT' + telemetry_url_test = 'file://///10.200.51.33/share/PHP' + + cm.add_row( + # :row_number=> show/increment row number + {:row_number=>true, :bgcolor=>bgcolor_os}, + {:text=>platform, :bgcolor=>bgcolor_os}, + {:text=>pass_rate.to_s+'%', :bgcolor=>bgcolor_test}, + {:text=>((cm.html?)?'<small>':'')+((change_pass_rate>0)?'+':'')+change_pass_rate.to_s+'%'+((cm.html?)?'</small>':''), :bgcolor=>(change_pass_rate>=0)?((change_pass_rate>0)?bgcolor_good:bgcolor_test):bgcolor_bad}, + {:text=>pass.to_s, :bgcolor=>bgcolor_test}, + {:text=>((cm.html?)?'<small>':'')+((change_pass>0)?'+':'')+change_pass.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_pass>=0)?((change_pass>0)?bgcolor_good:bgcolor_test):bgcolor_bad}, + {:text=>fail.to_s, :bgcolor=>bgcolor_test}, + {:text=>((cm.html?)?'<small>':'')+((change_fail>0)?'+':'')+change_fail.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_fail>=0)?((change_fail>0)?bgcolor_bad:bgcolor_test):bgcolor_good}, + {:text=>xfail_pass.to_s, :bgcolor=>bgcolor_test}, + {:text=>((cm.html?)?'<small>':'')+((change_xfail_pass>0)?'+':'')+change_xfail_pass.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_xfail_pass>=0)?((change_xfail_pass>0)?bgcolor_good:bgcolor_test):bgcolor_bad}, + {:text=>skip.to_s, :bgcolor=>bgcolor_skip}, + {:text=>((cm.html?)?'<small>':'')+((change_skip>0)?'+':'')+change_skip.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_skip>=0)?((change_skip>0)?:bgcolor_bad:bgcolor_skip):bgcolor_good}, + {:text=>xskip.to_s, :bgcolor=>bgcolor_skip}, + {:text=>((cm.html?)?'<small>':'')+((change_xskip>0)?'+':'')+change_xskip.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_xskip>=0)?((change_xskip>0)?bgcolor_bad:bgcolor_skip):bgcolor_good}, + {:text=>xfail_work.to_s, :bgcolor=>bgcolor_test_bug}, + {:text=>((cm.html?)?'<small>':'')+((change_xfail_work>0)?'+':'')+change_xfail_work.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_xfail_work>=0)?((change_xfail_work>0)?bgcolor_bad:bgcolor_test_bug):bgcolor_good}, + {:text=>bork.to_s, :bgcolor=>bgcolor_test_bug}, + {:text=>((cm.html?)?'<small>':'')+((change_bork>0)?'+':'')+change_bork.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_bork>=0)?((change_bork>0)?bgcolor_bad:bgcolor_test_bug):bgcolor_good}, + {:text=>unsupported.to_s, :bgcolor=>bgcolor_pftt_bug}, + {:text=>((cm.html?)?'<small>':'')+((change_unsupported>0)?'+':'')+change_unsupported.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_unsupported>=0)?((change_unsupported>0)?bgcolor_bad:bgcolor_pftt_bug):bgcolor_good}, + {:text=>(cm.html?)?"<a href=\"#{telemetry_url_base}\" target=\"_blank\">Base</a>":telemetry_url_base, :bgcolor=>bgcolor_telemetry}, + {:text=>(cm.html?)?"<a href=\"#{telemetry_url_test}\" target=\"_blank\">Test</a>":telemetry_url_test, :bgcolor=>bgcolor_telemetry} + ) + + } + end + } + + cm.add_row( + {:text=>(cm.html?)?'<strong>Windows INI:</strong>':'INI:', :colspan=>2}, + # TODO show INI + {:text=>ctx_set.inspect.to_s, :colspan=>20} + ) + cm.add_row( + {:text=>(cm.html?)?'<strong>Linux INI:</strong>':'INI:', :colspan=>2}, + # TODO show INI + {:text=>ctx_set.inspect.to_s, :colspan=>20} + ) + + cm.add_row( + {:text=>(cm.html?)?'<strong>Scenarios:</strong>':'Scenarios:', :colspan=>2}, + {:text=>ctx_set.inspect.to_s, :colspan=>20} + ) + + } + + return cm + end + + def write_attachments + attachments = {} + + # for each platform, attach a diff of each PASS, FAIL, SKIP, etc... list between each result set (test run) + resultsets_by_platform().each{|resultsets| + platform = resultsets[:platform] + arch = resultsets[:arch] + resultset_a = resultsets[:a] # single run of tests on one host + resultset_b = resultsets[:b] + + platform_file = platform.gsub(' ', '_')+'_'+arch + + attachments["#{platform_file}_PASS.diff"] = diff_file(resultset_a.list(:pass), resultset_b.list(:pass)) + attachments["#{platform_file}_FAIL.diff"] = diff_file(resultset_a.list(:fail), resultset_b.list(:fail)) + attachments["#{platform_file}_SKIP.diff"] = diff_file(resultset_a.list(:skip), resultset_b.list(:skip)) + # for the other lists, include diff even if its empty (so reader can tell there was no change) + # but to try to keep the number of attachments down (since there are 3-8 for each OS SKU (~30) => 90 to 240 attachments) + # readers will probably only be interested in the first 3 (PASS, FAIL, SKIP) + diff = diff_file(resultset_a.list(:xskip), resultset_b.list(:xskip)) + if diff.length > 0 + attachments["#{platform_file}_XSKIP.diff"] = diff_file(resultset_a.list(:xskip), resultset_b.list(:xskip)) + end + diff = diff_file(resultset_a.list(:works), resultset_b.list(:works)) + if diff.length > 0 + attachments["#{platform_file}_WORKS.diff"] = diff_file(resultset_a.list(:works), resultset_b.list(:works)) + end + diff = diff_file(resultset_a.list(:xfail), resultset_b.list(:xfail)) + if diff.length > 0 + attachments["#{platform_file}_XFAIL.diff"] = diff_file(resultset_a.list(:xfail), resultset_b.list(:xfail)) + end + diff = diff(resultset_a.list(:bork), resultset_b.list(:bork)) + if diff.length > 0 + attachments["#{platform_file}_BORK.diff"] = diff_file(resultset_a.list(:bork), resultset_b.list(:bork)) + end + diff = diff_file(resultset_a.list(:unsupported), resultset_b.list(:unsupported)) + if diff.length > 0 + attachments["#{platform_file}_UNSUPPORTED.diff"] = diff_file(resultset_a.list(:unsupported), resultset_b.list(:unsupported)) + end + } + # + + return attachments + end + + def write_html + html = write_intro + + table = Util::ColumnManager::Html.new() + write_table(table) + + html = html + table.render() + write_end + + return html + end + + def write_text + text = remove_html(write_intro()) + + table = Util::ColumnManager::Text.new(22) + write_table(table) + + text = text + "\r\n\r\n" + table.render() + "\r\n" + remove_html(write_end) + + return text + end + + end + end + end +end + \ No newline at end of file diff --git a/PFTT/lib/report/comparison/by_platform/func_release_comparison.rb b/PFTT/lib/report/comparison/by_platform/func_release_comparison.rb deleted file mode 100644 index 92205a1..0000000 --- a/PFTT/lib/report/comparison/by_platform/func_release_comparison.rb +++ /dev/null @@ -1,206 +0,0 @@ - -module Report - module Comparison - module ByPlatform - class FuncReleaseComparison < Base - - def title - 'Functional Release Comparison(FRC) Report' - end - - def write_intro - # TODO time, title - "<html><body><h1>#{title}</h1><p>Comparing @resultset_a.title (Base) with @resultset_b.title (Test) (using tests current as of @resultset_a.test_time</p><p>Click <a href=\"#{resultsets_comparison_url}\" target=\"_blank\">here</a> for a customizable comparison</p>" - end - - def write_end - "<p>* - tests missing required sections (test is not runnable)</p><p>** - tests with unsupported sections (test is not runnable)</p></body></html>" - end - - def write_table(cm) - # colors to use - bgcolor_os = '#f2f2f2' - bgcolor_test = '#ccff66' - bgcolor_skip = '#ffff00' - bgcolor_test_bug = '#ffc000' - bgcolor_pftt_bug = '#f28020' - bgcolor_telemetry = '#ccff66' - bgcolor_good = '#96dc28' - bgcolor_bad = '#ff0000' - # - - [['context_1', 'context_2'], ['context_1', 'context_3']].each{|ctx_set| - - ['IIS with WinCache', 'IIS', 'Command Line', 'Apache with APC', 'Apache with APC and IGBinary', 'Apache'].each{|mw_txt| - cm.add_row({:text=>(cm.html?)?'<strong>'+mw_txt+'</strong>':mw_txt, :colspan=>22, :center=>true}) - cm.add_row( - {:text=>'OS', :colspan=>2, :bgcolor=>bgcolor_os}, - {:text=>'', :colspan=>8, :bgcolor=>bgcolor_test}, - {:text=>'Skip', :colspan=>4, :bgcolor=>bgcolor_skip, :center=>true}, - {:text=>'Test Bug', :colspan=>4, :bgcolor=>bgcolor_test_bug, :center=>true}, - {:text=>'PFTT Bug', :colspan=>2, :bgcolor=>bgcolor_pftt_bug, :center=>true}, - {:text=>'Telemetry', :colspan=>2, :bgcolor=>bgcolor_telemetry, :center=>true} - ) - cm.add_row( - {:text=>'', :bgcolor=>bgcolor_os, :colspan=>2}, - {:text=>'Pass Rate', :bgcolor=>bgcolor_test, :colspan=>2, :center=>true}, - {:text=>'Pass', :bgcolor=>bgcolor_test, :colspan=>2, :center=>true}, - {:text=>'Fail', :bgcolor=>bgcolor_test, :colspan=>2, :center=>true}, - {:text=>'XFail(Pass)', :bgcolor=>bgcolor_test, :colspan=>2, :center=>true}, - {:text=>'Skip', :bgcolor=>bgcolor_skip, :colspan=>2, :center=>true}, - {:text=>'XSkip', :bgcolor=>bgcolor_skip, :colspan=>2, :center=>true}, - {:text=>'XFail(Work)', :bgcolor=>bgcolor_test_bug, :colspan=>2, :center=>true}, - {:text=>'Bork*', :bgcolor=>bgcolor_test_bug, :colspan=>2, :center=>true}, - {:text=>'Unsupported**', :bgcolor=>bgcolor_pftt_bug, :colspan=>2, :center=>true}, - {:text=>'', :bgcolor=>bgcolor_telemetry, :colspan=>2} - ) - - results = ['A Result'] - if results.empty? - cm.add_row({:text=>'No Results', :colspan=>22, :center=>true}) - else - results.each{|result| - platform = 'Win 2003r2 x86 SP0, SP1' - pass_rate = 50 - change_pass_rate = -5 - pass = 5000 - change_pass = -100 - fail = 5000 - change_fail = +50 - xfail_pass = 0 - change_xfail_pass = -10 - skip = 0 - change_skip = -100 - xskip = 0 - change_xskip = -100 - xfail_work = 0 - change_xfail_work = -100 - bork = 0 - change_bork = 0 - unsupported = 0 - change_unsupported = +10 - telemetry_url_base = 'file://///10.200.51.33/share/PFTT' - telemetry_url_test = 'file://///10.200.51.33/share/PHP' - - cm.add_row( - # :row_number=> show/increment row number - {:row_number=>true, :bgcolor=>bgcolor_os}, - {:text=>platform, :bgcolor=>bgcolor_os}, - {:text=>pass_rate.to_s+'%', :bgcolor=>bgcolor_test}, - {:text=>((cm.html?)?'<small>':'')+((change_pass_rate>0)?'+':'')+change_pass_rate.to_s+'%'+((cm.html?)?'</small>':''), :bgcolor=>(change_pass_rate>=0)?((change_pass_rate>0)?bgcolor_good:bgcolor_test):bgcolor_bad}, - {:text=>pass.to_s, :bgcolor=>bgcolor_test}, - {:text=>((cm.html?)?'<small>':'')+((change_pass>0)?'+':'')+change_pass.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_pass>=0)?((change_pass>0)?bgcolor_good:bgcolor_test):bgcolor_bad}, - {:text=>fail.to_s, :bgcolor=>bgcolor_test}, - {:text=>((cm.html?)?'<small>':'')+((change_fail>0)?'+':'')+change_fail.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_fail>=0)?((change_fail>0)?bgcolor_bad:bgcolor_test):bgcolor_good}, - {:text=>xfail_pass.to_s, :bgcolor=>bgcolor_test}, - {:text=>((cm.html?)?'<small>':'')+((change_xfail_pass>0)?'+':'')+change_xfail_pass.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_xfail_pass>=0)?((change_xfail_pass>0)?bgcolor_good:bgcolor_test):bgcolor_bad}, - {:text=>skip.to_s, :bgcolor=>bgcolor_skip}, - {:text=>((cm.html?)?'<small>':'')+((change_skip>0)?'+':'')+change_skip.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_skip>=0)?((change_skip>0)?:bgcolor_bad:bgcolor_skip):bgcolor_good}, - {:text=>xskip.to_s, :bgcolor=>bgcolor_skip}, - {:text=>((cm.html?)?'<small>':'')+((change_xskip>0)?'+':'')+change_xskip.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_xskip>=0)?((change_xskip>0)?bgcolor_bad:bgcolor_skip):bgcolor_good}, - {:text=>xfail_work.to_s, :bgcolor=>bgcolor_test_bug}, - {:text=>((cm.html?)?'<small>':'')+((change_xfail_work>0)?'+':'')+change_xfail_work.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_xfail_work>=0)?((change_xfail_work>0)?bgcolor_bad:bgcolor_test_bug):bgcolor_good}, - {:text=>bork.to_s, :bgcolor=>bgcolor_test_bug}, - {:text=>((cm.html?)?'<small>':'')+((change_bork>0)?'+':'')+change_bork.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_bork>=0)?((change_bork>0)?bgcolor_bad:bgcolor_test_bug):bgcolor_good}, - {:text=>unsupported.to_s, :bgcolor=>bgcolor_pftt_bug}, - {:text=>((cm.html?)?'<small>':'')+((change_unsupported>0)?'+':'')+change_unsupported.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_unsupported>=0)?((change_unsupported>0)?bgcolor_bad:bgcolor_pftt_bug):bgcolor_good}, - {:text=>(cm.html?)?"<a href=\"#{telemetry_url_base}\" target=\"_blank\">Base</a>":telemetry_url_base, :bgcolor=>bgcolor_telemetry}, - {:text=>(cm.html?)?"<a href=\"#{telemetry_url_test}\" target=\"_blank\">Test</a>":telemetry_url_test, :bgcolor=>bgcolor_telemetry} - ) - - } - end - } - - cm.add_row( - {:text=>(cm.html?)?'<strong>Windows INI:</strong>':'INI:', :colspan=>2}, - # TODO show INI - {:text=>ctx_set.inspect.to_s, :colspan=>20} - ) - cm.add_row( - {:text=>(cm.html?)?'<strong>Linux INI:</strong>':'INI:', :colspan=>2}, - # TODO show INI - {:text=>ctx_set.inspect.to_s, :colspan=>20} - ) - - cm.add_row( - {:text=>(cm.html?)?'<strong>Scenarios:</strong>':'Scenarios:', :colspan=>2}, - {:text=>ctx_set.inspect.to_s, :colspan=>20} - ) - - } - - return cm - end - - def write_attachments - attachments = {} - - # for each platform, attach a diff of each PASS, FAIL, SKIP, etc... list between each result set (test run) - resultsets_by_platform().each{|resultsets| - platform = resultsets[:platform] - arch = resultsets[:arch] - resultset_a = resultsets[:a] # single run of tests on one host - resultset_b = resultsets[:b] - - platform_file = platform.gsub(' ', '_')+'_'+arch - - attachments["#{platform_file}_PASS.diff"] = diff_file(resultset_a.list(:pass), resultset_b.list(:pass)) - attachments["#{platform_file}_FAIL.diff"] = diff_file(resultset_a.list(:fail), resultset_b.list(:fail)) - attachments["#{platform_file}_SKIP.diff"] = diff_file(resultset_a.list(:skip), resultset_b.list(:skip)) - # for the other lists, include diff even if its empty (so reader can tell there was no change) - # but to try to keep the number of attachments down (since there are 3-8 for each OS SKU (~30) => 90 to 240 attachments) - # readers will probably only be interested in the first 3 (PASS, FAIL, SKIP) - diff = diff_file(resultset_a.list(:xskip), resultset_b.list(:xskip)) - if diff.length > 0 - attachments["#{platform_file}_XSKIP.diff"] = diff_file(resultset_a.list(:xskip), resultset_b.list(:xskip)) - end - diff = diff_file(resultset_a.list(:works), resultset_b.list(:works)) - if diff.length > 0 - attachments["#{platform_file}_WORKS.diff"] = diff_file(resultset_a.list(:works), resultset_b.list(:works)) - end - diff = diff_file(resultset_a.list(:xfail), resultset_b.list(:xfail)) - if diff.length > 0 - attachments["#{platform_file}_XFAIL.diff"] = diff_file(resultset_a.list(:xfail), resultset_b.list(:xfail)) - end - diff = diff(resultset_a.list(:bork), resultset_b.list(:bork)) - if diff.length > 0 - attachments["#{platform_file}_BORK.diff"] = diff_file(resultset_a.list(:bork), resultset_b.list(:bork)) - end - diff = diff_file(resultset_a.list(:unsupported), resultset_b.list(:unsupported)) - if diff.length > 0 - attachments["#{platform_file}_UNSUPPORTED.diff"] = diff_file(resultset_a.list(:unsupported), resultset_b.list(:unsupported)) - end - } - # - - return attachments - end - - def write_html - html = write_intro - - table = Util::ColumnManager::Html.new() - write_table(table) - - html = html + table.render() + write_end - - return html - end - - def write_text - text = remove_html(write_intro()) - - table = Util::ColumnManager::Text.new(22) - write_table(table) - - text = text + "\r\n\r\n" + table.render() + "\r\n" + remove_html(write_end) - - return text - end - - end - end - end -end - \ No newline at end of file diff --git a/PFTT/lib/report/comparison/by_platform/perf_build_comparison.rb b/PFTT/lib/report/comparison/by_platform/perf_build_comparison.rb new file mode 100644 index 0000000..f73954d --- /dev/null +++ b/PFTT/lib/report/comparison/by_platform/perf_build_comparison.rb @@ -0,0 +1,106 @@ + +module Report + module Comparison + module ByPlatform + class PerfBuildComparison < Base + + def title + 'Performance Build Comparison(PBC) Report' + end + + def write_intro + # TODO time, title + "<html><body><h1>#{title}</h1><p>Comparing performance of @resultset_a.title (Base) with @resultset_b.title (Test)</p><p>Click <a href=\"#{resultsets_comparison_url}\" target=\"_blank\">here</a> for a customizable comparison</p>" + end + + def write_end + "</body></html>" + end + + def write_app_detail_table(app, cm) + cm.add_row({:text=>app, :colspan=>8}) + cm.add_row('', 'OS', 'vrtu', 'tps', 'kcpt', 'bpt', 'cpu', 'err') + + return cm + end + + def write_comparison_table(app, cm, r) + r[TypedToken::ArrayToken::ScenarioSet].each{|scn_set| + + cm.add_row( + {:text=>(cm.html?)?'<strong>Scenarios:</strong>':'Scenarios:', :colspan=>2}, + {:text=>scn_set.inspect.to_s, :colspan=>20} + ) + + cm.add_row({:text=>'', :colspan=>4}, {:text=>'IIS', :colspan=>6}, {:text=>'Apache', :colspan=>9}) + cm.add_row({:text=>'', :colspan=>4}, {:text=>'Load Agents', :colspan=>2}, {:text=>'No WinCache', :colspan=>3}, {:text=>'WinCache', :colspan=>3}, {:text=>'No APC', :colspan=>3}, {:text=>'APC', :colspan=>3}, {:text=>'APC w/ IGBinary', :colspan=>3}) + cm.add_row('', 'OS', 'Physical', 'Virtual', 'Base', 'Test', 'Gain', 'Base', 'Test', 'Gain', 'Base', 'Test', 'Gain', 'Base', 'Test', 'Gain') + + windows_ini, posix_ini = nil + r[scn_set, TypedToken::StringToken::OSName].each{|results| + # TODO + cm.add_row( + {:row_number=>true}, + os_short_name(results.os), + results.physical_client, + results.virtual_client, + '50', '100', '50%', '100', '50', '50%', '50', '100', '50%', '100', '50', '50%') + + windows_ini = results.windows_ini + posix_ini = results.posix_ini + } + + cm.add_row( + {:text=>(cm.html?)?'<strong>Windows INI:</strong>':'INI:', :colspan=>2}, + {:text=>windows_ini.to_s, :colspan=>20} + ) + cm.add_row( + {:text=>(cm.html?)?'<strong>POSIX INI:</strong>':'INI:', :colspan=>2}, + {:text=>posix_ini.to_s, :colspan=>20} + ) + + + + } + + return cm + end + + def write_tables(cm) + html_txt = '' + apps = @results_ctx[TypedToken::StringToken::AppName]#['Hello World', 'Typo3', 'Drupal', 'MediaWiki', 'Wordpress', 'Joomla'] + apps.keys.each{|app| + html_txt += write_comparison_table(app, cm.clone, apps).render() + } + + apps.keys.each{|app| + html_txt += write_app_detail_table(app, cm.clone, apps).render() + } + return html_txt + end + + def write_html + html = write_intro + + table = Util::ColumnManager::Html.new() + + html = html + write_tables(table) + write_end + + return html + end + + def write_text + text = remove_html(write_intro()) + + table = Util::ColumnManager::Text.new(22) + + text = text + "\r\n\r\n" + write_tables(table) + "\r\n" + remove_html(write_end) + + return text + end + + end + end + end +end + \ No newline at end of file diff --git a/PFTT/lib/report/comparison/by_platform/perf_release_comparison.rb b/PFTT/lib/report/comparison/by_platform/perf_release_comparison.rb deleted file mode 100644 index 43ab751..0000000 --- a/PFTT/lib/report/comparison/by_platform/perf_release_comparison.rb +++ /dev/null @@ -1,106 +0,0 @@ - -module Report - module Comparison - module ByPlatform - class PerfReleaseComparison < Base - - def title - 'Performance Release Comparison(PRC) Report' - end - - def write_intro - # TODO time, title - "<html><body><h1>#{title}</h1><p>Comparing performance of @resultset_a.title (Base) with @resultset_b.title (Test)</p><p>Click <a href=\"#{resultsets_comparison_url}\" target=\"_blank\">here</a> for a customizable comparison</p>" - end - - def write_end - "</body></html>" - end - - def write_app_detail_table(app, cm) - cm.add_row({:text=>app, :colspan=>8}) - cm.add_row('', 'OS', 'vrtu', 'tps', 'kcpt', 'bpt', 'cpu', 'err') - - return cm - end - - def write_comparison_table(app, cm, r) - r[TypedToken::ArrayToken::ScenarioSet].each{|scn_set| - - cm.add_row( - {:text=>(cm.html?)?'<strong>Scenarios:</strong>':'Scenarios:', :colspan=>2}, - {:text=>scn_set.inspect.to_s, :colspan=>20} - ) - - cm.add_row({:text=>'', :colspan=>4}, {:text=>'IIS', :colspan=>6}, {:text=>'Apache', :colspan=>9}) - cm.add_row({:text=>'', :colspan=>4}, {:text=>'Load Agents', :colspan=>2}, {:text=>'No WinCache', :colspan=>3}, {:text=>'WinCache', :colspan=>3}, {:text=>'No APC', :colspan=>3}, {:text=>'APC', :colspan=>3}, {:text=>'APC w/ IGBinary', :colspan=>3}) - cm.add_row('', 'OS', 'Physical', 'Virtual', 'Base', 'Test', 'Gain', 'Base', 'Test', 'Gain', 'Base', 'Test', 'Gain', 'Base', 'Test', 'Gain') - - windows_ini, posix_ini = nil - r[scn_set, TypedToken::StringToken::OSName].each{|results| - # TODO - cm.add_row( - {:row_number=>true}, - os_short_name(results.os), - results.physical_client, - results.virtual_client, - '50', '100', '50%', '100', '50', '50%', '50', '100', '50%', '100', '50', '50%') - - windows_ini = results.windows_ini - posix_ini = results.posix_ini - } - - cm.add_row( - {:text=>(cm.html?)?'<strong>Windows INI:</strong>':'INI:', :colspan=>2}, - {:text=>windows_ini.to_s, :colspan=>20} - ) - cm.add_row( - {:text=>(cm.html?)?'<strong>POSIX INI:</strong>':'INI:', :colspan=>2}, - {:text=>posix_ini.to_s, :colspan=>20} - ) - - - - } - - return cm - end - - def write_tables(cm) - html_txt = '' - apps = @results_ctx[TypedToken::StringToken::AppName]#['Hello World', 'Typo3', 'Drupal', 'MediaWiki', 'Wordpress', 'Joomla'] - apps.keys.each{|app| - html_txt += write_comparison_table(app, cm.clone, apps).render() - } - - apps.keys.each{|app| - html_txt += write_app_detail_table(app, cm.clone, apps).render() - } - return html_txt - end - - def write_html - html = write_intro - - table = Util::ColumnManager::Html.new() - - html = html + write_tables(table) + write_end - - return html - end - - def write_text - text = remove_html(write_intro()) - - table = Util::ColumnManager::Text.new(22) - - text = text + "\r\n\r\n" + write_tables(table) + "\r\n" + remove_html(write_end) - - return text - end - - end - end - end -end - \ No newline at end of file diff --git a/PFTT/lib/server/fbcd_mailer.rb b/PFTT/lib/server/fbcd_mailer.rb new file mode 100644 index 0000000..fc2ec73 --- /dev/null +++ b/PFTT/lib/server/fbcd_mailer.rb @@ -0,0 +1,24 @@ + +module Server + class FBCDMailer + def initialize + @downloaded = false + end + def execute + if downloaded or not ( hours / 24.0 == hours / 24 ) + return + end + + # check every hour, on the hour, until a new snapshot is downloaded for that day + # at which point, wait until the next day (24h-hour when snapshot was downloaded) + sg = SnapshotGetter.new + sg.ensure_latest_snapshot + + # if 24 hours+ have elapsed since last snapshot, test this one again anyway (test against the + # snapshot from before that) + downloaded = sg.is_new? or not ( hours / 24.0 == hours / 24 ) + + # TODO generate+send FRC-B + end + end +end \ No newline at end of file diff --git a/PFTT/lib/server/lock_server.rb b/PFTT/lib/server/lock_server.rb new file mode 100644 index 0000000..058e147 --- /dev/null +++ b/PFTT/lib/server/lock_server.rb @@ -0,0 +1,80 @@ + +module Server + $lock_timeout = '10 MINUTES' + $lock_cmd = nil # LATER CLI command to set WTT status => Busy + $release_cmd = nil # LATER CLI command to set WTT status => Ready + + class LockServer + def terastation + ['\\terastation\share', 'test', 'test'] + end + def github + [] + end + def pftt_mysql_server + ['10.200.50.51', 'root', 'password01!'] + end + # + def dep_server + # + terastation + end + def phpt_server + # + terastation + end + def phpbuild_server + # + terastation + end + def database_server + # + pftt_mysql_server # also the server that runs _pftt_server.rb (doesn't have to be though) + end + def config_server + # + terastation + end + def update_server + # + github + end + def lock(host_name) + # check if host is locked or if lock has not expired + + @db.query("UPDATE hosts SET lock_id=RAND() AND lock_renew_time=DATE_ADD(NOW(), #{$lock_timeout}) WHERE host_name LIKE ? AND ( lock_id=NULL OR lock_renew_time=NULL OR DATE_ADD(lock_renew_time, #{$lock_timeout}) < NOW() )") + + lock_id = @db['lock_id'] # TODO + + if lock_id + if $lock_cmd + exec($lock_cmd) + end + return lock_id + else + raise XMLRPC::FaultException.new(-1, "Host not found or host is already locked") + end + end + def lock_renew(host_name, lock_id) + # update lock expiration time + + @db.execute("UPDATE hosts WHERE lock_id=? AND host_name=? SET lock_renew_time=NOW()", lock_id, host_name) + end + def release(host_name, lock_id) + # unlock the host + + @db.execute("UPDATE hosts WHERE lock_id=? AND host_name=? SET lock_id=NULL", lock_id, host_name) + + if $release_cmd + exec($release_cmd) + end + end + def view + # return list of hosts + + @db.execute("SELECT * FROM hosts") + end + + # TODO @db.execute("CREATE TABLE IF NOT EXISTS hosts(host_id INTEGER PRIMARY KEY AUTO_INCREMENT, host_name VARCHAR(20), status ENUM('LOCKED', 'READY'), os_sku VARCHAR(20), arch ENUM('x86', 'x64'), ip_address VARCHAR(20), lock_id INTEGER, lock_renew_time DATETIME)") + end +end \ No newline at end of file diff --git a/PFTT/lib/server/network_monitor.rb b/PFTT/lib/server/network_monitor.rb new file mode 100644 index 0000000..4e3892d --- /dev/null +++ b/PFTT/lib/server/network_monitor.rb @@ -0,0 +1,22 @@ + +module Server + class NetworkMonitor + def execute + # send Report::Network every 6 hours + do_report = hours / 6.0 == hours / 6 + + # check every host every hour. include in network report + # send report immediately if host is down + hosts.each do |host| + unless host.alive? + do_report = true + end + end + + if do_report + report = Report::Network.new() + report.email() + end + end + end +end \ No newline at end of file diff --git a/PFTT/lib/server/psb.rb b/PFTT/lib/server/psb.rb new file mode 100644 index 0000000..0584335 --- /dev/null +++ b/PFTT/lib/server/psb.rb @@ -0,0 +1,122 @@ + +require 'webrick' + +module Web + class PSB < WEBrick::HTTPServlet::AbstractServlet + + def do_fbc_report(response, base, test) + # flip base & test around to make sure base is always older than test + + report = Report::Comparison::ByPlatform::FuncBuildComparison.new() + report.html_print + end + + def do_fbc_list(response, base=nil) + # list all functional test runs (if base!=nil, then this is to choose second run) + + +#Run +#Start Time +#Hosts +#Builds + end + + def do_GET(request, response) + response.status = 200 + response['Content-Type'] = 'text/html' + + do_header(response) + if request.query['report'] == 'fbc' + if request.query['base'] + if request.query['test'] + do_fbc_report(response, request.query['base'], request.query['test']) + else + # show the func run list again, but + # have the user pick the second run to compare to + do_fbc_list(response, request.query['base']) + end + else + do_fbc_list(response) + end + elsif request.query['report'] == 'pbc' + # LATER PSB support for PBC reports + response.body += "<p>PBC reporting not yet implemented</p>" + else + do_home(response) + end + do_footer(response) + end + + def do_home(response) + response.body += <<-HOME +<h3>To start, choose Functional or Performance reporting</h3> + +<p><a href="https://github.com/OSTC/PFTT2" target="_blank"><font color="blue">PFTT Project</font></a></p> +<br/> +<h2>Terminology</h2> +<ul> +<li><strong>PSB</strong> - PFTT Statisics Browser</li> +<li><strong>FBC</strong> - Functional Build Comparison Report +<ul> +<li><strong>FBC-D</strong> - For Daily use. One scenario set tested.</li> +<li><strong>FBC-S</strong> - For Stable release. All scenarios tested.</li> +</ul> +</li> +<li><strong>PBC</strong> - Performance Build Comparison Report +<ul> +<li><strong>PBC-D</strong> - For Daily use. One scenario set tested.</li> +<li><strong>PBC-S</strong> - For Stable release. All scenarios tested.</li> +</ul> +<li><strong>PFTT</strong> +<ul> +<li><strong>PFTT Server</strong> - Coordinates Clients access to Hosts</li> +<li><strong>PFTT Client</strong> - Executes tests on hosts.</li> +<li><strong>PFTT Host</strong> - Machines that are tested.</li> +</ul> +<li><strong>FR</strong> - Functional Run Report (Functional Testing all OS/Build/Middlwares)</li> +<li><strong>FCR</strong> - Functional Combination (Host/Build/Middleware) Run Report</li> +<li><strong>PR</strong> - Performance Run Report (Performance Testing all OS/Build/Middlwares)</li> +<li><strong>PCR</strong> - Performance Combination (Host/Build/Middlware) Run Report</li> +</ul> +HOME + end + + def do_header(response) + response.body = <<-HEAD +<html> +<head> +<title>PFTT Statisics Browser</title> +</head> +<body> +<h1>PSB: PFTT Statistics Browser</h1> + +<table> +<tr> +<td><a href="/"><font color="blue">Home</font></a></td> +<td><a href="/?report=fbc"><font color="blue">Functional (FBC)</font></a></td> +<td><a href="/?report=pbc"><font color="blue">Performance (PBC)</font></a></td> +</tr> +</table> + + +HEAD + end + + def do_footer(response) + response.body += <<-FOOT + +<table> +<tr> +<td><a href="/"><font color="blue">Home</font></a></td> +<td><a href="/?report=fbc"><font color="blue">Functional (FBC)</font></a></td> +<td><a href="/?report=pbc"><font color="blue">Performance (PBC)</font></a></td> +</tr> +</table> + +</body> +</html> +FOOT + end + + end # end class PSB +end # end module Web diff --git a/PFTT/lib/server/snapshot_getter.rb b/PFTT/lib/server/snapshot_getter.rb new file mode 100644 index 0000000..6de21ad --- /dev/null +++ b/PFTT/lib/server/snapshot_getter.rb @@ -0,0 +1,158 @@ + +module Server + class SnapshotGetter + def initialize + @revision_index = "http://windows.php.net/downloads/snaps/php-5.4/" # MUST have trailing / + end + + # + # may raise exceptions + def ensure_latest_snapshot + urls = scrape_snapshot_urls() + + threads = [] + urls.each do |url| + t = Thread.start do + download_file_to_local(host, url) + end + threads.push(t) + end + + # wait for them to all finish + threads.each do |t| + t.join + end + + return ['local paths to decompressed latest snapshots'] # TODO + end + + protected + + def scrape_snapshot_urls() + rev_index_url = URI.parse(@revision_index) + + # URL goes to a page that shows a list of revisions ('Revision Index') + # find the newest URL, then go to that page (the 'Snapshot Index') + # Snapshot index lists several files + # find php-5.4-ts, nts, src, and provide those URLs + # + # http_response = Net::HTTP.get_response(rev_index_url) + # + # if http_response.code != '200' + # raise 'HttpErrorRevisionIndex' + # end + # + # doc = Nokogiri::HTML(http_response.body) + # + # hrefs = doc.xpath("*//a/@href") + # + # newest_snap_href = nil + # hrefs.each do |attribute| + # href = attribute.value + # # make sure its not some link to something else that got added to this page at some point + # if href.starts_with?(rev_index_url.path+'r') + # newest_snap_href = href + # end + # end + # + # unless newest_snap_href + # raise 'SnapshotLinkNotFoundError' + # end + # + newest_snap_href = "downloads/snaps/php-5.4/r318303/" # TODO temp + puts newest_snap_href.inspect + + http_response = Net::HTTP.get_response(URI.parse("http://"+rev_index_url.host+"/"+newest_snap_href)) + + if http_response.code != '200' + raise 'HttpErrorSnapshotIndex' + end + + doc = Nokogiri::HTML(http_response.body) + + hrefs = doc.xpath("*//a") + + php_nts_url, php_ts_url, php_src_url = nil + + hrefs.each do |a_tag| + href = a_tag.attributes['href'].value + unless href.starts_with?('http://') + href = "http://"+(rev_index_url.host+"/"+href).gsub('//', '/') + end + + text = ((a_tag.children.empty?)?'':a_tag.children.first.to_s) + + if text.starts_with?('php-5.4-nts') + php_nts_url = href + elsif text.starts_with?('php-5.4-ts') + php_ts_url = href + elsif text.starts_with?('php-5.4-src') + php_src_url = href + end + + end + + unless php_nts_url and php_ts_url + raise 'PHPBuildLinksNotFound' + end + puts [php_nts_url, php_ts_url, php_src_url].inspect + return [php_nts_url, php_ts_url, php_src_url] + end # def scrape_snapshot_urls + + def download_file_to_local(host, remote_url) + local_filename = 'C:/PFTT-PHPS/'+File.basename(remote_url) # TODO add temp dir path + local_dir = local_filename + + # local dir should be same as local filename of ZIP without the .zip + if local_dir.ends_with?('.zip') + local_dir = local_dir[0..local_dir.length-1-'.zip'.length] + end + + unless File.exist?(local_dir) + # not decompressed + + unless File.exist?(local_filename) + # zip not downloaded + + remote_url = URI.parse(remote_url) + + puts remote_url.inspect + + localhost.exec!("C:/php-sdk/bin/wget -O #{local_filename} #{remote_url}") # TODO + # Net::HTTP.start(remote_url.host, remote_url.port) do |http| + # begin + # file = open(local_filename, 'wb') + # http.request_get(remote_url.path) do |resp| + # resp.read_body do |segment| + # file.write(segment) + # #sleep 10 # hack to wait for buffer to refill + # end + # end + # ensure + # file.close + # end + # end + end + + # then decompress + # + if local_filename.include?('php-5.4-src') or local_filename.include?('php-5.5-src') + # php src already comes compressed into a directory (NTS and TS don't) + local_dir = "C:/PFTT-PHPS" + end + + # decompress into a directory named with the filename + localhost.mkdir(local_dir) + + # run unzip from that directory + localhost.exec!("C:/php-sdk/bin/unzip #{local_filename}", {:chdir=>local_dir}) + end + + puts local_filename + + return local_filename + + end + + end # class SnapshotGetter +end diff --git a/PFTT/lib/server/svn_updater.rb b/PFTT/lib/server/svn_updater.rb new file mode 100644 index 0000000..5d722fd --- /dev/null +++ b/PFTT/lib/server/svn_updater.rb @@ -0,0 +1,12 @@ + +module Server + class SVNUpdater + def execute + localhost.exec!("svn checkout https://svn.php.net/repository/php/php-src", {:chdir=>"c:/php-sdk/svn/"}) + localhost.exec!("move php-src\branches .", {:chdir=>"c:/php-sdk/svn/"}) + localhost.exec!("move php-src\trunk .", {:chdir=>"c:/php-sdk/svn/"}) + localhost.exec!("move php-src\tags .", {:chdir=>"c:/php-sdk/svn/"}) + localhost.exec!("svn update branches\PHP_5_4 branches\PHP_5_3", {:chdir=>"c:/php-sdk/svn/"}) + end + end +end \ No newline at end of file
diff --git a/PFTT/lib/report/comparison/by_platform/func_build_comparison.rb b/PFTT/lib/report/comparison/by_platform/func_build_comparison.rb new file mode 100644 index 0000000..2501647 --- /dev/null +++ b/PFTT/lib/report/comparison/by_platform/func_build_comparison.rb @@ -0,0 +1,206 @@ + +module Report + module Comparison + module ByPlatform + class FuncBuildComparison < Base + + def title + 'Functional Build Comparison(FBC) Report' + end + + def write_intro + # TODO time, title + "<html><body><h1>#{title}</h1><p>Comparing @resultset_a.title (Base) with @resultset_b.title (Test) (using tests current as of @resultset_a.test_time</p><p>Click <a href=\"#{resultsets_comparison_url}\" target=\"_blank\">here</a> for a customizable comparison</p>" + end + + def write_end + "<p>* - tests missing required sections (test is not runnable)</p><p>** - tests with unsupported sections (test is not runnable)</p></body></html>" + end + + def write_table(cm) + # colors to use + bgcolor_os = '#f2f2f2' + bgcolor_test = '#ccff66' + bgcolor_skip = '#ffff00' + bgcolor_test_bug = '#ffc000' + bgcolor_pftt_bug = '#f28020' + bgcolor_telemetry = '#ccff66' + bgcolor_good = '#96dc28' + bgcolor_bad = '#ff0000' + # + + [['context_1', 'context_2'], ['context_1', 'context_3']].each{|ctx_set| + + ['IIS with WinCache', 'IIS', 'Command Line', 'Apache with APC', 'Apache with APC and IGBinary', 'Apache'].each{|mw_txt| + cm.add_row({:text=>(cm.html?)?'<strong>'+mw_txt+'</strong>':mw_txt, :colspan=>22, :center=>true}) + cm.add_row( + {:text=>'OS', :colspan=>2, :bgcolor=>bgcolor_os}, + {:text=>'', :colspan=>8, :bgcolor=>bgcolor_test}, + {:text=>'Skip', :colspan=>4, :bgcolor=>bgcolor_skip, :center=>true}, + {:text=>'Test Bug', :colspan=>4, :bgcolor=>bgcolor_test_bug, :center=>true}, + {:text=>'PFTT Bug', :colspan=>2, :bgcolor=>bgcolor_pftt_bug, :center=>true}, + {:text=>'Telemetry', :colspan=>2, :bgcolor=>bgcolor_telemetry, :center=>true} + ) + cm.add_row( + {:text=>'', :bgcolor=>bgcolor_os, :colspan=>2}, + {:text=>'Pass Rate', :bgcolor=>bgcolor_test, :colspan=>2, :center=>true}, + {:text=>'Pass', :bgcolor=>bgcolor_test, :colspan=>2, :center=>true}, + {:text=>'Fail', :bgcolor=>bgcolor_test, :colspan=>2, :center=>true}, + {:text=>'XFail(Pass)', :bgcolor=>bgcolor_test, :colspan=>2, :center=>true}, + {:text=>'Skip', :bgcolor=>bgcolor_skip, :colspan=>2, :center=>true}, + {:text=>'XSkip', :bgcolor=>bgcolor_skip, :colspan=>2, :center=>true}, + {:text=>'XFail(Work)', :bgcolor=>bgcolor_test_bug, :colspan=>2, :center=>true}, + {:text=>'Bork*', :bgcolor=>bgcolor_test_bug, :colspan=>2, :center=>true}, + {:text=>'Unsupported**', :bgcolor=>bgcolor_pftt_bug, :colspan=>2, :center=>true}, + {:text=>'', :bgcolor=>bgcolor_telemetry, :colspan=>2} + ) + + results = ['A Result'] + if results.empty? + cm.add_row({:text=>'No Results', :colspan=>22, :center=>true}) + else + results.each{|result| + platform = 'Win 2003r2 x86 SP0, SP1' + pass_rate = 50 + change_pass_rate = -5 + pass = 5000 + change_pass = -100 + fail = 5000 + change_fail = +50 + xfail_pass = 0 + change_xfail_pass = -10 + skip = 0 + change_skip = -100 + xskip = 0 + change_xskip = -100 + xfail_work = 0 + change_xfail_work = -100 + bork = 0 + change_bork = 0 + unsupported = 0 + change_unsupported = +10 + telemetry_url_base = 'file://///10.200.51.33/share/PFTT' + telemetry_url_test = 'file://///10.200.51.33/share/PHP' + + cm.add_row( + # :row_number=> show/increment row number + {:row_number=>true, :bgcolor=>bgcolor_os}, + {:text=>platform, :bgcolor=>bgcolor_os}, + {:text=>pass_rate.to_s+'%', :bgcolor=>bgcolor_test}, + {:text=>((cm.html?)?'<small>':'')+((change_pass_rate>0)?'+':'')+change_pass_rate.to_s+'%'+((cm.html?)?'</small>':''), :bgcolor=>(change_pass_rate>=0)?((change_pass_rate>0)?bgcolor_good:bgcolor_test):bgcolor_bad}, + {:text=>pass.to_s, :bgcolor=>bgcolor_test}, + {:text=>((cm.html?)?'<small>':'')+((change_pass>0)?'+':'')+change_pass.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_pass>=0)?((change_pass>0)?bgcolor_good:bgcolor_test):bgcolor_bad}, + {:text=>fail.to_s, :bgcolor=>bgcolor_test}, + {:text=>((cm.html?)?'<small>':'')+((change_fail>0)?'+':'')+change_fail.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_fail>=0)?((change_fail>0)?bgcolor_bad:bgcolor_test):bgcolor_good}, + {:text=>xfail_pass.to_s, :bgcolor=>bgcolor_test}, + {:text=>((cm.html?)?'<small>':'')+((change_xfail_pass>0)?'+':'')+change_xfail_pass.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_xfail_pass>=0)?((change_xfail_pass>0)?bgcolor_good:bgcolor_test):bgcolor_bad}, + {:text=>skip.to_s, :bgcolor=>bgcolor_skip}, + {:text=>((cm.html?)?'<small>':'')+((change_skip>0)?'+':'')+change_skip.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_skip>=0)?((change_skip>0)?:bgcolor_bad:bgcolor_skip):bgcolor_good}, + {:text=>xskip.to_s, :bgcolor=>bgcolor_skip}, + {:text=>((cm.html?)?'<small>':'')+((change_xskip>0)?'+':'')+change_xskip.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_xskip>=0)?((change_xskip>0)?bgcolor_bad:bgcolor_skip):bgcolor_good}, + {:text=>xfail_work.to_s, :bgcolor=>bgcolor_test_bug}, + {:text=>((cm.html?)?'<small>':'')+((change_xfail_work>0)?'+':'')+change_xfail_work.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_xfail_work>=0)?((change_xfail_work>0)?bgcolor_bad:bgcolor_test_bug):bgcolor_good}, + {:text=>bork.to_s, :bgcolor=>bgcolor_test_bug}, + {:text=>((cm.html?)?'<small>':'')+((change_bork>0)?'+':'')+change_bork.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_bork>=0)?((change_bork>0)?bgcolor_bad:bgcolor_test_bug):bgcolor_good}, + {:text=>unsupported.to_s, :bgcolor=>bgcolor_pftt_bug}, + {:text=>((cm.html?)?'<small>':'')+((change_unsupported>0)?'+':'')+change_unsupported.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_unsupported>=0)?((change_unsupported>0)?bgcolor_bad:bgcolor_pftt_bug):bgcolor_good}, + {:text=>(cm.html?)?"<a href=\"#{telemetry_url_base}\" target=\"_blank\">Base</a>":telemetry_url_base, :bgcolor=>bgcolor_telemetry}, + {:text=>(cm.html?)?"<a href=\"#{telemetry_url_test}\" target=\"_blank\">Test</a>":telemetry_url_test, :bgcolor=>bgcolor_telemetry} + ) + + } + end + } + + cm.add_row( + {:text=>(cm.html?)?'<strong>Windows INI:</strong>':'INI:', :colspan=>2}, + # TODO show INI + {:text=>ctx_set.inspect.to_s, :colspan=>20} + ) + cm.add_row( + {:text=>(cm.html?)?'<strong>Linux INI:</strong>':'INI:', :colspan=>2}, + # TODO show INI + {:text=>ctx_set.inspect.to_s, :colspan=>20} + ) + + cm.add_row( + {:text=>(cm.html?)?'<strong>Scenarios:</strong>':'Scenarios:', :colspan=>2}, + {:text=>ctx_set.inspect.to_s, :colspan=>20} + ) + + } + + return cm + end + + def write_attachments + attachments = {} + + # for each platform, attach a diff of each PASS, FAIL, SKIP, etc... list between each result set (test run) + resultsets_by_platform().each{|resultsets| + platform = resultsets[:platform] + arch = resultsets[:arch] + resultset_a = resultsets[:a] # single run of tests on one host + resultset_b = resultsets[:b] + + platform_file = platform.gsub(' ', '_')+'_'+arch + + attachments["#{platform_file}_PASS.diff"] = diff_file(resultset_a.list(:pass), resultset_b.list(:pass)) + attachments["#{platform_file}_FAIL.diff"] = diff_file(resultset_a.list(:fail), resultset_b.list(:fail)) + attachments["#{platform_file}_SKIP.diff"] = diff_file(resultset_a.list(:skip), resultset_b.list(:skip)) + # for the other lists, include diff even if its empty (so reader can tell there was no change) + # but to try to keep the number of attachments down (since there are 3-8 for each OS SKU (~30) => 90 to 240 attachments) + # readers will probably only be interested in the first 3 (PASS, FAIL, SKIP) + diff = diff_file(resultset_a.list(:xskip), resultset_b.list(:xskip)) + if diff.length > 0 + attachments["#{platform_file}_XSKIP.diff"] = diff_file(resultset_a.list(:xskip), resultset_b.list(:xskip)) + end + diff = diff_file(resultset_a.list(:works), resultset_b.list(:works)) + if diff.length > 0 + attachments["#{platform_file}_WORKS.diff"] = diff_file(resultset_a.list(:works), resultset_b.list(:works)) + end + diff = diff_file(resultset_a.list(:xfail), resultset_b.list(:xfail)) + if diff.length > 0 + attachments["#{platform_file}_XFAIL.diff"] = diff_file(resultset_a.list(:xfail), resultset_b.list(:xfail)) + end + diff = diff(resultset_a.list(:bork), resultset_b.list(:bork)) + if diff.length > 0 + attachments["#{platform_file}_BORK.diff"] = diff_file(resultset_a.list(:bork), resultset_b.list(:bork)) + end + diff = diff_file(resultset_a.list(:unsupported), resultset_b.list(:unsupported)) + if diff.length > 0 + attachments["#{platform_file}_UNSUPPORTED.diff"] = diff_file(resultset_a.list(:unsupported), resultset_b.list(:unsupported)) + end + } + # + + return attachments + end + + def write_html + html = write_intro + + table = Util::ColumnManager::Html.new() + write_table(table) + + html = html + table.render() + write_end + + return html + end + + def write_text + text = remove_html(write_intro()) + + table = Util::ColumnManager::Text.new(22) + write_table(table) + + text = text + "\r\n\r\n" + table.render() + "\r\n" + remove_html(write_end) + + return text + end + + end + end + end +end + \ No newline at end of file diff --git a/PFTT/lib/report/comparison/by_platform/func_release_comparison.rb b/PFTT/lib/report/comparison/by_platform/func_release_comparison.rb deleted file mode 100644 index 92205a1..0000000 --- a/PFTT/lib/report/comparison/by_platform/func_release_comparison.rb +++ /dev/null @@ -1,206 +0,0 @@ - -module Report - module Comparison - module ByPlatform - class FuncReleaseComparison < Base - - def title - 'Functional Release Comparison(FRC) Report' - end - - def write_intro - # TODO time, title - "<html><body><h1>#{title}</h1><p>Comparing @resultset_a.title (Base) with @resultset_b.title (Test) (using tests current as of @resultset_a.test_time</p><p>Click <a href=\"#{resultsets_comparison_url}\" target=\"_blank\">here</a> for a customizable comparison</p>" - end - - def write_end - "<p>* - tests missing required sections (test is not runnable)</p><p>** - tests with unsupported sections (test is not runnable)</p></body></html>" - end - - def write_table(cm) - # colors to use - bgcolor_os = '#f2f2f2' - bgcolor_test = '#ccff66' - bgcolor_skip = '#ffff00' - bgcolor_test_bug = '#ffc000' - bgcolor_pftt_bug = '#f28020' - bgcolor_telemetry = '#ccff66' - bgcolor_good = '#96dc28' - bgcolor_bad = '#ff0000' - # - - [['context_1', 'context_2'], ['context_1', 'context_3']].each{|ctx_set| - - ['IIS with WinCache', 'IIS', 'Command Line', 'Apache with APC', 'Apache with APC and IGBinary', 'Apache'].each{|mw_txt| - cm.add_row({:text=>(cm.html?)?'<strong>'+mw_txt+'</strong>':mw_txt, :colspan=>22, :center=>true}) - cm.add_row( - {:text=>'OS', :colspan=>2, :bgcolor=>bgcolor_os}, - {:text=>'', :colspan=>8, :bgcolor=>bgcolor_test}, - {:text=>'Skip', :colspan=>4, :bgcolor=>bgcolor_skip, :center=>true}, - {:text=>'Test Bug', :colspan=>4, :bgcolor=>bgcolor_test_bug, :center=>true}, - {:text=>'PFTT Bug', :colspan=>2, :bgcolor=>bgcolor_pftt_bug, :center=>true}, - {:text=>'Telemetry', :colspan=>2, :bgcolor=>bgcolor_telemetry, :center=>true} - ) - cm.add_row( - {:text=>'', :bgcolor=>bgcolor_os, :colspan=>2}, - {:text=>'Pass Rate', :bgcolor=>bgcolor_test, :colspan=>2, :center=>true}, - {:text=>'Pass', :bgcolor=>bgcolor_test, :colspan=>2, :center=>true}, - {:text=>'Fail', :bgcolor=>bgcolor_test, :colspan=>2, :center=>true}, - {:text=>'XFail(Pass)', :bgcolor=>bgcolor_test, :colspan=>2, :center=>true}, - {:text=>'Skip', :bgcolor=>bgcolor_skip, :colspan=>2, :center=>true}, - {:text=>'XSkip', :bgcolor=>bgcolor_skip, :colspan=>2, :center=>true}, - {:text=>'XFail(Work)', :bgcolor=>bgcolor_test_bug, :colspan=>2, :center=>true}, - {:text=>'Bork*', :bgcolor=>bgcolor_test_bug, :colspan=>2, :center=>true}, - {:text=>'Unsupported**', :bgcolor=>bgcolor_pftt_bug, :colspan=>2, :center=>true}, - {:text=>'', :bgcolor=>bgcolor_telemetry, :colspan=>2} - ) - - results = ['A Result'] - if results.empty? - cm.add_row({:text=>'No Results', :colspan=>22, :center=>true}) - else - results.each{|result| - platform = 'Win 2003r2 x86 SP0, SP1' - pass_rate = 50 - change_pass_rate = -5 - pass = 5000 - change_pass = -100 - fail = 5000 - change_fail = +50 - xfail_pass = 0 - change_xfail_pass = -10 - skip = 0 - change_skip = -100 - xskip = 0 - change_xskip = -100 - xfail_work = 0 - change_xfail_work = -100 - bork = 0 - change_bork = 0 - unsupported = 0 - change_unsupported = +10 - telemetry_url_base = 'file://///10.200.51.33/share/PFTT' - telemetry_url_test = 'file://///10.200.51.33/share/PHP' - - cm.add_row( - # :row_number=> show/increment row number - {:row_number=>true, :bgcolor=>bgcolor_os}, - {:text=>platform, :bgcolor=>bgcolor_os}, - {:text=>pass_rate.to_s+'%', :bgcolor=>bgcolor_test}, - {:text=>((cm.html?)?'<small>':'')+((change_pass_rate>0)?'+':'')+change_pass_rate.to_s+'%'+((cm.html?)?'</small>':''), :bgcolor=>(change_pass_rate>=0)?((change_pass_rate>0)?bgcolor_good:bgcolor_test):bgcolor_bad}, - {:text=>pass.to_s, :bgcolor=>bgcolor_test}, - {:text=>((cm.html?)?'<small>':'')+((change_pass>0)?'+':'')+change_pass.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_pass>=0)?((change_pass>0)?bgcolor_good:bgcolor_test):bgcolor_bad}, - {:text=>fail.to_s, :bgcolor=>bgcolor_test}, - {:text=>((cm.html?)?'<small>':'')+((change_fail>0)?'+':'')+change_fail.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_fail>=0)?((change_fail>0)?bgcolor_bad:bgcolor_test):bgcolor_good}, - {:text=>xfail_pass.to_s, :bgcolor=>bgcolor_test}, - {:text=>((cm.html?)?'<small>':'')+((change_xfail_pass>0)?'+':'')+change_xfail_pass.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_xfail_pass>=0)?((change_xfail_pass>0)?bgcolor_good:bgcolor_test):bgcolor_bad}, - {:text=>skip.to_s, :bgcolor=>bgcolor_skip}, - {:text=>((cm.html?)?'<small>':'')+((change_skip>0)?'+':'')+change_skip.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_skip>=0)?((change_skip>0)?:bgcolor_bad:bgcolor_skip):bgcolor_good}, - {:text=>xskip.to_s, :bgcolor=>bgcolor_skip}, - {:text=>((cm.html?)?'<small>':'')+((change_xskip>0)?'+':'')+change_xskip.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_xskip>=0)?((change_xskip>0)?bgcolor_bad:bgcolor_skip):bgcolor_good}, - {:text=>xfail_work.to_s, :bgcolor=>bgcolor_test_bug}, - {:text=>((cm.html?)?'<small>':'')+((change_xfail_work>0)?'+':'')+change_xfail_work.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_xfail_work>=0)?((change_xfail_work>0)?bgcolor_bad:bgcolor_test_bug):bgcolor_good}, - {:text=>bork.to_s, :bgcolor=>bgcolor_test_bug}, - {:text=>((cm.html?)?'<small>':'')+((change_bork>0)?'+':'')+change_bork.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_bork>=0)?((change_bork>0)?bgcolor_bad:bgcolor_test_bug):bgcolor_good}, - {:text=>unsupported.to_s, :bgcolor=>bgcolor_pftt_bug}, - {:text=>((cm.html?)?'<small>':'')+((change_unsupported>0)?'+':'')+change_unsupported.to_s+((cm.html?)?'</small>':''), :bgcolor=>(change_unsupported>=0)?((change_unsupported>0)?bgcolor_bad:bgcolor_pftt_bug):bgcolor_good}, - {:text=>(cm.html?)?"<a href=\"#{telemetry_url_base}\" target=\"_blank\">Base</a>":telemetry_url_base, :bgcolor=>bgcolor_telemetry}, - {:text=>(cm.html?)?"<a href=\"#{telemetry_url_test}\" target=\"_blank\">Test</a>":telemetry_url_test, :bgcolor=>bgcolor_telemetry} - ) - - } - end - } - - cm.add_row( - {:text=>(cm.html?)?'<strong>Windows INI:</strong>':'INI:', :colspan=>2}, - # TODO show INI - {:text=>ctx_set.inspect.to_s, :colspan=>20} - ) - cm.add_row( - {:text=>(cm.html?)?'<strong>Linux INI:</strong>':'INI:', :colspan=>2}, - # TODO show INI - {:text=>ctx_set.inspect.to_s, :colspan=>20} - ) - - cm.add_row( - {:text=>(cm.html?)?'<strong>Scenarios:</strong>':'Scenarios:', :colspan=>2}, - {:text=>ctx_set.inspect.to_s, :colspan=>20} - ) - - } - - return cm - end - - def write_attachments - attachments = {} - - # for each platform, attach a diff of each PASS, FAIL, SKIP, etc... list between each result set (test run) - resultsets_by_platform().each{|resultsets| - platform = resultsets[:platform] - arch = resultsets[:arch] - resultset_a = resultsets[:a] # single run of tests on one host - resultset_b = resultsets[:b] - - platform_file = platform.gsub(' ', '_')+'_'+arch - - attachments["#{platform_file}_PASS.diff"] = diff_file(resultset_a.list(:pass), resultset_b.list(:pass)) - attachments["#{platform_file}_FAIL.diff"] = diff_file(resultset_a.list(:fail), resultset_b.list(:fail)) - attachments["#{platform_file}_SKIP.diff"] = diff_file(resultset_a.list(:skip), resultset_b.list(:skip)) - # for the other lists, include diff even if its empty (so reader can tell there was no change) - # but to try to keep the number of attachments down (since there are 3-8 for each OS SKU (~30) => 90 to 240 attachments) - # readers will probably only be interested in the first 3 (PASS, FAIL, SKIP) - diff = diff_file(resultset_a.list(:xskip), resultset_b.list(:xskip)) - if diff.length > 0 - attachments["#{platform_file}_XSKIP.diff"] = diff_file(resultset_a.list(:xskip), resultset_b.list(:xskip)) - end - diff = diff_file(resultset_a.list(:works), resultset_b.list(:works)) - if diff.length > 0 - attachments["#{platform_file}_WORKS.diff"] = diff_file(resultset_a.list(:works), resultset_b.list(:works)) - end - diff = diff_file(resultset_a.list(:xfail), resultset_b.list(:xfail)) - if diff.length > 0 - attachments["#{platform_file}_XFAIL.diff"] = diff_file(resultset_a.list(:xfail), resultset_b.list(:xfail)) - end - diff = diff(resultset_a.list(:bork), resultset_b.list(:bork)) - if diff.length > 0 - attachments["#{platform_file}_BORK.diff"] = diff_file(resultset_a.list(:bork), resultset_b.list(:bork)) - end - diff = diff_file(resultset_a.list(:unsupported), resultset_b.list(:unsupported)) - if diff.length > 0 - attachments["#{platform_file}_UNSUPPORTED.diff"] = diff_file(resultset_a.list(:unsupported), resultset_b.list(:unsupported)) - end - } - # - - return attachments - end - - def write_html - html = write_intro - - table = Util::ColumnManager::Html.new() - write_table(table) - - html = html + table.render() + write_end - - return html - end - - def write_text - text = remove_html(write_intro()) - - table = Util::ColumnManager::Text.new(22) - write_table(table) - - text = text + "\r\n\r\n" + table.render() + "\r\n" + remove_html(write_end) - - return text - end - - end - end - end -end - \ No newline at end of file diff --git a/PFTT/lib/report/comparison/by_platform/perf_build_comparison.rb b/PFTT/lib/report/comparison/by_platform/perf_build_comparison.rb new file mode 100644 index 0000000..f73954d --- /dev/null +++ b/PFTT/lib/report/comparison/by_platform/perf_build_comparison.rb @@ -0,0 +1,106 @@ + +module Report + module Comparison + module ByPlatform + class PerfBuildComparison < Base + + def title + 'Performance Build Comparison(PBC) Report' + end + + def write_intro + # TODO time, title + "<html><body><h1>#{title}</h1><p>Comparing performance of @resultset_a.title (Base) with @resultset_b.title (Test)</p><p>Click <a href=\"#{resultsets_comparison_url}\" target=\"_blank\">here</a> for a customizable comparison</p>" + end + + def write_end + "</body></html>" + end + + def write_app_detail_table(app, cm) + cm.add_row({:text=>app, :colspan=>8}) + cm.add_row('', 'OS', 'vrtu', 'tps', 'kcpt', 'bpt', 'cpu', 'err') + + return cm + end + + def write_comparison_table(app, cm, r) + r[TypedToken::ArrayToken::ScenarioSet].each{|scn_set| + + cm.add_row( + {:text=>(cm.html?)?'<strong>Scenarios:</strong>':'Scenarios:', :colspan=>2}, + {:text=>scn_set.inspect.to_s, :colspan=>20} + ) + + cm.add_row({:text=>'', :colspan=>4}, {:text=>'IIS', :colspan=>6}, {:text=>'Apache', :colspan=>9}) + cm.add_row({:text=>'', :colspan=>4}, {:text=>'Load Agents', :colspan=>2}, {:text=>'No WinCache', :colspan=>3}, {:text=>'WinCache', :colspan=>3}, {:text=>'No APC', :colspan=>3}, {:text=>'APC', :colspan=>3}, {:text=>'APC w/ IGBinary', :colspan=>3}) + cm.add_row('', 'OS', 'Physical', 'Virtual', 'Base', 'Test', 'Gain', 'Base', 'Test', 'Gain', 'Base', 'Test', 'Gain', 'Base', 'Test', 'Gain') + + windows_ini, posix_ini = nil + r[scn_set, TypedToken::StringToken::OSName].each{|results| + # TODO + cm.add_row( + {:row_number=>true}, + os_short_name(results.os), + results.physical_client, + results.virtual_client, + '50', '100', '50%', '100', '50', '50%', '50', '100', '50%', '100', '50', '50%') + + windows_ini = results.windows_ini + posix_ini = results.posix_ini + } + + cm.add_row( + {:text=>(cm.html?)?'<strong>Windows INI:</strong>':'INI:', :colspan=>2}, + {:text=>windows_ini.to_s, :colspan=>20} + ) + cm.add_row( + {:text=>(cm.html?)?'<strong>POSIX INI:</strong>':'INI:', :colspan=>2}, + {:text=>posix_ini.to_s, :colspan=>20} + ) + + + + } + + return cm + end + + def write_tables(cm) + html_txt = '' + apps = @results_ctx[TypedToken::StringToken::AppName]#['Hello World', 'Typo3', 'Drupal', 'MediaWiki', 'Wordpress', 'Joomla'] + apps.keys.each{|app| + html_txt += write_comparison_table(app, cm.clone, apps).render() + } + + apps.keys.each{|app| + html_txt += write_app_detail_table(app, cm.clone, apps).render() + } + return html_txt + end + + def write_html + html = write_intro + + table = Util::ColumnManager::Html.new() + + html = html + write_tables(table) + write_end + + return html + end + + def write_text + text = remove_html(write_intro()) + + table = Util::ColumnManager::Text.new(22) + + text = text + "\r\n\r\n" + write_tables(table) + "\r\n" + remove_html(write_end) + + return text + end + + end + end + end +end + \ No newline at end of file diff --git a/PFTT/lib/report/comparison/by_platform/perf_release_comparison.rb b/PFTT/lib/report/comparison/by_platform/perf_release_comparison.rb deleted file mode 100644 index 43ab751..0000000 --- a/PFTT/lib/report/comparison/by_platform/perf_release_comparison.rb +++ /dev/null @@ -1,106 +0,0 @@ - -module Report - module Comparison - module ByPlatform - class PerfReleaseComparison < Base - - def title - 'Performance Release Comparison(PRC) Report' - end - - def write_intro - # TODO time, title - "<html><body><h1>#{title}</h1><p>Comparing performance of @resultset_a.title (Base) with @resultset_b.title (Test)</p><p>Click <a href=\"#{resultsets_comparison_url}\" target=\"_blank\">here</a> for a customizable comparison</p>" - end - - def write_end - "</body></html>" - end - - def write_app_detail_table(app, cm) - cm.add_row({:text=>app, :colspan=>8}) - cm.add_row('', 'OS', 'vrtu', 'tps', 'kcpt', 'bpt', 'cpu', 'err') - - return cm - end - - def write_comparison_table(app, cm, r) - r[TypedToken::ArrayToken::ScenarioSet].each{|scn_set| - - cm.add_row( - {:text=>(cm.html?)?'<strong>Scenarios:</strong>':'Scenarios:', :colspan=>2}, - {:text=>scn_set.inspect.to_s, :colspan=>20} - ) - - cm.add_row({:text=>'', :colspan=>4}, {:text=>'IIS', :colspan=>6}, {:text=>'Apache', :colspan=>9}) - cm.add_row({:text=>'', :colspan=>4}, {:text=>'Load Agents', :colspan=>2}, {:text=>'No WinCache', :colspan=>3}, {:text=>'WinCache', :colspan=>3}, {:text=>'No APC', :colspan=>3}, {:text=>'APC', :colspan=>3}, {:text=>'APC w/ IGBinary', :colspan=>3}) - cm.add_row('', 'OS', 'Physical', 'Virtual', 'Base', 'Test', 'Gain', 'Base', 'Test', 'Gain', 'Base', 'Test', 'Gain', 'Base', 'Test', 'Gain') - - windows_ini, posix_ini = nil - r[scn_set, TypedToken::StringToken::OSName].each{|results| - # TODO - cm.add_row( - {:row_number=>true}, - os_short_name(results.os), - results.physical_client, - results.virtual_client, - '50', '100', '50%', '100', '50', '50%', '50', '100', '50%', '100', '50', '50%') - - windows_ini = results.windows_ini - posix_ini = results.posix_ini - } - - cm.add_row( - {:text=>(cm.html?)?'<strong>Windows INI:</strong>':'INI:', :colspan=>2}, - {:text=>windows_ini.to_s, :colspan=>20} - ) - cm.add_row( - {:text=>(cm.html?)?'<strong>POSIX INI:</strong>':'INI:', :colspan=>2}, - {:text=>posix_ini.to_s, :colspan=>20} - ) - - - - } - - return cm - end - - def write_tables(cm) - html_txt = '' - apps = @results_ctx[TypedToken::StringToken::AppName]#['Hello World', 'Typo3', 'Drupal', 'MediaWiki', 'Wordpress', 'Joomla'] - apps.keys.each{|app| - html_txt += write_comparison_table(app, cm.clone, apps).render() - } - - apps.keys.each{|app| - html_txt += write_app_detail_table(app, cm.clone, apps).render() - } - return html_txt - end - - def write_html - html = write_intro - - table = Util::ColumnManager::Html.new() - - html = html + write_tables(table) + write_end - - return html - end - - def write_text - text = remove_html(write_intro()) - - table = Util::ColumnManager::Text.new(22) - - text = text + "\r\n\r\n" + write_tables(table) + "\r\n" + remove_html(write_end) - - return text - end - - end - end - end -end - \ No newline at end of file diff --git a/PFTT/lib/server/fbcd_mailer.rb b/PFTT/lib/server/fbcd_mailer.rb new file mode 100644 index 0000000..fc2ec73 --- /dev/null +++ b/PFTT/lib/server/fbcd_mailer.rb @@ -0,0 +1,24 @@ + +module Server + class FBCDMailer + def initialize + @downloaded = false + end + def execute + if downloaded or not ( hours / 24.0 == hours / 24 ) + return + end + + # check every hour, on the hour, until a new snapshot is downloaded for that day + # at which point, wait until the next day (24h-hour when snapshot was downloaded) + sg = SnapshotGetter.new + sg.ensure_latest_snapshot + + # if 24 hours+ have elapsed since last snapshot, test this one again anyway (test against the + # snapshot from before that) + downloaded = sg.is_new? or not ( hours / 24.0 == hours / 24 ) + + # TODO generate+send FRC-B + end + end +end \ No newline at end of file diff --git a/PFTT/lib/server/lock_server.rb b/PFTT/lib/server/lock_server.rb new file mode 100644 index 0000000..058e147 --- /dev/null +++ b/PFTT/lib/server/lock_server.rb @@ -0,0 +1,80 @@ + +module Server + $lock_timeout = '10 MINUTES' + $lock_cmd = nil # LATER CLI command to set WTT status => Busy + $release_cmd = nil # LATER CLI command to set WTT status => Ready + + class LockServer + def terastation + ['\\terastation\share', 'test', 'test'] + end + def github + [] + end + def pftt_mysql_server + ['10.200.50.51', 'root', 'password01!'] + end + # + def dep_server + # + terastation + end + def phpt_server + # + terastation + end + def phpbuild_server + # + terastation + end + def database_server + # + pftt_mysql_server # also the server that runs _pftt_server.rb (doesn't have to be though) + end + def config_server + # + terastation + end + def update_server + # + github + end + def lock(host_name) + # check if host is locked or if lock has not expired + + @db.query("UPDATE hosts SET lock_id=RAND() AND lock_renew_time=DATE_ADD(NOW(), #{$lock_timeout}) WHERE host_name LIKE ? AND ( lock_id=NULL OR lock_renew_time=NULL OR DATE_ADD(lock_renew_time, #{$lock_timeout}) < NOW() )") + + lock_id = @db['lock_id'] # TODO + + if lock_id + if $lock_cmd + exec($lock_cmd) + end + return lock_id + else + raise XMLRPC::FaultException.new(-1, "Host not found or host is already locked") + end + end + def lock_renew(host_name, lock_id) + # update lock expiration time + + @db.execute("UPDATE hosts WHERE lock_id=? AND host_name=? SET lock_renew_time=NOW()", lock_id, host_name) + end + def release(host_name, lock_id) + # unlock the host + + @db.execute("UPDATE hosts WHERE lock_id=? AND host_name=? SET lock_id=NULL", lock_id, host_name) + + if $release_cmd + exec($release_cmd) + end + end + def view + # return list of hosts + + @db.execute("SELECT * FROM hosts") + end + + # TODO @db.execute("CREATE TABLE IF NOT EXISTS hosts(host_id INTEGER PRIMARY KEY AUTO_INCREMENT, host_name VARCHAR(20), status ENUM('LOCKED', 'READY'), os_sku VARCHAR(20), arch ENUM('x86', 'x64'), ip_address VARCHAR(20), lock_id INTEGER, lock_renew_time DATETIME)") + end +end \ No newline at end of file diff --git a/PFTT/lib/server/network_monitor.rb b/PFTT/lib/server/network_monitor.rb new file mode 100644 index 0000000..4e3892d --- /dev/null +++ b/PFTT/lib/server/network_monitor.rb @@ -0,0 +1,22 @@ + +module Server + class NetworkMonitor + def execute + # send Report::Network every 6 hours + do_report = hours / 6.0 == hours / 6 + + # check every host every hour. include in network report + # send report immediately if host is down + hosts.each do |host| + unless host.alive? + do_report = true + end + end + + if do_report + report = Report::Network.new() + report.email() + end + end + end +end \ No newline at end of file diff --git a/PFTT/lib/server/psb.rb b/PFTT/lib/server/psb.rb new file mode 100644 index 0000000..0584335 --- /dev/null +++ b/PFTT/lib/server/psb.rb @@ -0,0 +1,122 @@ + +require 'webrick' + +module Web + class PSB < WEBrick::HTTPServlet::AbstractServlet + + def do_fbc_report(response, base, test) + # flip base & test around to make sure base is always older than test + + report = Report::Comparison::ByPlatform::FuncBuildComparison.new() + report.html_print + end + + def do_fbc_list(response, base=nil) + # list all functional test runs (if base!=nil, then this is to choose second run) + + +#Run +#Start Time +#Hosts +#Builds + end + + def do_GET(request, response) + response.status = 200 + response['Content-Type'] = 'text/html' + + do_header(response) + if request.query['report'] == 'fbc' + if request.query['base'] + if request.query['test'] + do_fbc_report(response, request.query['base'], request.query['test']) + else + # show the func run list again, but + # have the user pick the second run to compare to + do_fbc_list(response, request.query['base']) + end + else + do_fbc_list(response) + end + elsif request.query['report'] == 'pbc' + # LATER PSB support for PBC reports + response.body += "<p>PBC reporting not yet implemented</p>" + else + do_home(response) + end + do_footer(response) + end + + def do_home(response) + response.body += <<-HOME +<h3>To start, choose Functional or Performance reporting</h3> + +<p><a href="https://github.com/OSTC/PFTT2" target="_blank"><font color="blue">PFTT Project</font></a></p> +<br/> +<h2>Terminology</h2> +<ul> +<li><strong>PSB</strong> - PFTT Statisics Browser</li> +<li><strong>FBC</strong> - Functional Build Comparison Report +<ul> +<li><strong>FBC-D</strong> - For Daily use. One scenario set tested.</li> +<li><strong>FBC-S</strong> - For Stable release. All scenarios tested.</li> +</ul> +</li> +<li><strong>PBC</strong> - Performance Build Comparison Report +<ul> +<li><strong>PBC-D</strong> - For Daily use. One scenario set tested.</li> +<li><strong>PBC-S</strong> - For Stable release. All scenarios tested.</li> +</ul> +<li><strong>PFTT</strong> +<ul> +<li><strong>PFTT Server</strong> - Coordinates Clients access to Hosts</li> +<li><strong>PFTT Client</strong> - Executes tests on hosts.</li> +<li><strong>PFTT Host</strong> - Machines that are tested.</li> +</ul> +<li><strong>FR</strong> - Functional Run Report (Functional Testing all OS/Build/Middlwares)</li> +<li><strong>FCR</strong> - Functional Combination (Host/Build/Middleware) Run Report</li> +<li><strong>PR</strong> - Performance Run Report (Performance Testing all OS/Build/Middlwares)</li> +<li><strong>PCR</strong> - Performance Combination (Host/Build/Middlware) Run Report</li> +</ul> +HOME + end + + def do_header(response) + response.body = <<-HEAD +<html> +<head> +<title>PFTT Statisics Browser</title> +</head> +<body> +<h1>PSB: PFTT Statistics Browser</h1> + +<table> +<tr> +<td><a href="/"><font color="blue">Home</font></a></td> +<td><a href="/?report=fbc"><font color="blue">Functional (FBC)</font></a></td> +<td><a href="/?report=pbc"><font color="blue">Performance (PBC)</font></a></td> +</tr> +</table> + + +HEAD + end + + def do_footer(response) + response.body += <<-FOOT + +<table> +<tr> +<td><a href="/"><font color="blue">Home</font></a></td> +<td><a href="/?report=fbc"><font color="blue">Functional (FBC)</font></a></td> +<td><a href="/?report=pbc"><font color="blue">Performance (PBC)</font></a></td> +</tr> +</table> + +</body> +</html> +FOOT + end + + end # end class PSB +end # end module Web diff --git a/PFTT/lib/server/snapshot_getter.rb b/PFTT/lib/server/snapshot_getter.rb new file mode 100644 index 0000000..6de21ad --- /dev/null +++ b/PFTT/lib/server/snapshot_getter.rb @@ -0,0 +1,158 @@ + +module Server + class SnapshotGetter + def initialize + @revision_index = "http://windows.php.net/downloads/snaps/php-5.4/" # MUST have trailing / + end + + # + # may raise exceptions + def ensure_latest_snapshot + urls = scrape_snapshot_urls() + + threads = [] + urls.each do |url| + t = Thread.start do + download_file_to_local(host, url) + end + threads.push(t) + end + + # wait for them to all finish + threads.each do |t| + t.join + end + + return ['local paths to decompressed latest snapshots'] # TODO + end + + protected + + def scrape_snapshot_urls() + rev_index_url = URI.parse(@revision_index) + + # URL goes to a page that shows a list of revisions ('Revision Index') + # find the newest URL, then go to that page (the 'Snapshot Index') + # Snapshot index lists several files + # find php-5.4-ts, nts, src, and provide those URLs + # + # http_response = Net::HTTP.get_response(rev_index_url) + # + # if http_response.code != '200' + # raise 'HttpErrorRevisionIndex' + # end + # + # doc = Nokogiri::HTML(http_response.body) + # + # hrefs = doc.xpath("*//a/@href") + # + # newest_snap_href = nil + # hrefs.each do |attribute| + # href = attribute.value + # # make sure its not some link to something else that got added to this page at some point + # if href.starts_with?(rev_index_url.path+'r') + # newest_snap_href = href + # end + # end + # + # unless newest_snap_href + # raise 'SnapshotLinkNotFoundError' + # end + # + newest_snap_href = "downloads/snaps/php-5.4/r318303/" # TODO temp + puts newest_snap_href.inspect + + http_response = Net::HTTP.get_response(URI.parse("http://"+rev_index_url.host+"/"+newest_snap_href)) + + if http_response.code != '200' + raise 'HttpErrorSnapshotIndex' + end + + doc = Nokogiri::HTML(http_response.body) + + hrefs = doc.xpath("*//a") + + php_nts_url, php_ts_url, php_src_url = nil + + hrefs.each do |a_tag| + href = a_tag.attributes['href'].value + unless href.starts_with?('http://') + href = "http://"+(rev_index_url.host+"/"+href).gsub('//', '/') + end + + text = ((a_tag.children.empty?)?'':a_tag.children.first.to_s) + + if text.starts_with?('php-5.4-nts') + php_nts_url = href + elsif text.starts_with?('php-5.4-ts') + php_ts_url = href + elsif text.starts_with?('php-5.4-src') + php_src_url = href + end + + end + + unless php_nts_url and php_ts_url + raise 'PHPBuildLinksNotFound' + end + puts [php_nts_url, php_ts_url, php_src_url].inspect + return [php_nts_url, php_ts_url, php_src_url] + end # def scrape_snapshot_urls + + def download_file_to_local(host, remote_url) + local_filename = 'C:/PFTT-PHPS/'+File.basename(remote_url) # TODO add temp dir path + local_dir = local_filename + + # local dir should be same as local filename of ZIP without the .zip + if local_dir.ends_with?('.zip') + local_dir = local_dir[0..local_dir.length-1-'.zip'.length] + end + + unless File.exist?(local_dir) + # not decompressed + + unless File.exist?(local_filename) + # zip not downloaded + + remote_url = URI.parse(remote_url) + + puts remote_url.inspect + + localhost.exec!("C:/php-sdk/bin/wget -O #{local_filename} #{remote_url}") # TODO + # Net::HTTP.start(remote_url.host, remote_url.port) do |http| + # begin + # file = open(local_filename, 'wb') + # http.request_get(remote_url.path) do |resp| + # resp.read_body do |segment| + # file.write(segment) + # #sleep 10 # hack to wait for buffer to refill + # end + # end + # ensure + # file.close + # end + # end + end + + # then decompress + # + if local_filename.include?('php-5.4-src') or local_filename.include?('php-5.5-src') + # php src already comes compressed into a directory (NTS and TS don't) + local_dir = "C:/PFTT-PHPS" + end + + # decompress into a directory named with the filename + localhost.mkdir(local_dir) + + # run unzip from that directory + localhost.exec!("C:/php-sdk/bin/unzip #{local_filename}", {:chdir=>local_dir}) + end + + puts local_filename + + return local_filename + + end + + end # class SnapshotGetter +end diff --git a/PFTT/lib/server/svn_updater.rb b/PFTT/lib/server/svn_updater.rb new file mode 100644 index 0000000..5d722fd --- /dev/null +++ b/PFTT/lib/server/svn_updater.rb @@ -0,0 +1,12 @@ + +module Server + class SVNUpdater + def execute + localhost.exec!("svn checkout https://svn.php.net/repository/php/php-src", {:chdir=>"c:/php-sdk/svn/"}) + localhost.exec!("move php-src\branches .", {:chdir=>"c:/php-sdk/svn/"}) + localhost.exec!("move php-src\trunk .", {:chdir=>"c:/php-sdk/svn/"}) + localhost.exec!("move php-src\tags .", {:chdir=>"c:/php-sdk/svn/"}) + localhost.exec!("svn update branches\PHP_5_4 branches\PHP_5_3", {:chdir=>"c:/php-sdk/svn/"}) + end + end +end \ No newline at end of file