com pftt2: reworked result storage: PFTT/_pftt.rb PFTT/li b/diff.rb PFTT/lib/host.rb PFTT/lib/host/local.rb P FTT/lib/host/remote/ssh.rb
PFTT/lib/middleware/cli.rb P FTT/lib/middleware/http.rb PFTT/lib/middleware/http/apache .rb PFTT/lib/middleware/http/apache/mod_php.rb PFTT/l
ib/middleware/http/apache/mod_php/apc.rb PFTT/lib/middlew are/http/apache/mod_php/apc/igbinary.rb PFTT/lib/middlewa re/http/iis.rb
PFTT/lib/middleware/http/iis/fast_cgi.rb PFTT/l

Commit: 6749585dbfe6ebb38d8638eadd771dbef730aab0 Author: Matt Ficken <v-mafick@.redmond.corp.microsoft.com> Thu, 20 Oct 2011 19:35:44 -0700 Parents: eda8059019287d70e3a405e36884ab113f6841f0 Branches: master Link: http://git.php.net/?p=pftt2.git;a=commitdiff;h=6749585dbfe6ebb38d8638eadd771dbef730aab0 Log: reworked result storage Changed paths: M PFTT/_pftt.rb M PFTT/lib/diff.rb M PFTT/lib/host.rb M PFTT/lib/host/local.rb M PFTT/lib/host/remote/ssh.rb M PFTT/lib/middleware/cli.rb M PFTT/lib/middleware/http.rb M PFTT/lib/middleware/http/apache.rb M PFTT/lib/middleware/http/apache/mod_php.rb M PFTT/lib/middleware/http/apache/mod_php/apc.rb M PFTT/lib/middleware/http/apache/mod_php/apc/igbinary.rb M PFTT/lib/middleware/http/iis.rb M PFTT/lib/middleware/http/iis/fast_cgi.rb M PFTT/lib/middleware/http/iis/fast_cgi/wincache.rb M PFTT/lib/pftt_client.rb M PFTT/lib/php_build.rb M PFTT/lib/phpt_test_result.rb M PFTT/lib/report/comparison/by_platform/func_release_comparison.rb M PFTT/lib/report/comparison/by_platform/perf_release_comparison.rb M PFTT/lib/report/network.rb M PFTT/lib/report/run/per_host.rb M PFTT/lib/report/run/per_host/per_build/per_middleware.rb M PFTT/lib/report/run/per_host/per_build/per_middleware/func.rb M PFTT/lib/test_bench.rb M PFTT/lib/test_bench/phpt.rb M PFTT/lib/test_bench/wcat.rb M PFTT/lib/test_result/perf_result.rb M PFTT/lib/util.rb M PFTT/lib/util/column_manager.rb M PFTT/lib/util/column_manager/html.rb M PFTT/lib/util/column_manager/text.rb M PFTT/lib/util/install.rb M PFTT/scripts/SDK4Win/PHP M PFTT/scripts/SDK4Win/SetupPHPSDK.cmd

diff --git a/PFTT/_pftt.rb b/PFTT/_pftt.rb index 09632f0..f0bec8d 100644 --- a/PFTT/_pftt.rb +++ b/PFTT/_pftt.rb @@ -77,6 +77,7 @@ $brief_output = false $skip_none = false $force_deploy = false $debug_test = false +$auto_triage = false $html_report = false $interactive_mode = false $is_windows = RbConfig::CONFIG['host_os'].include?('mingw') or RbConfig::CONFIG['host_os'].include?('mswin') @@ -94,7 +95,6 @@ class PfttOptions < OptionsHash exit(4) end - # TODO options.replace(YAML::load( File.open(default_config_file) )) unless !File.exists?( default_config_file ) options[:action] = args[0] @@ -210,9 +210,6 @@ class PfttOptions < OptionsHash exit(4) end - # TODO does it make sense to load config file here?? - #options.replace(YAML::load( File.open(default_config_file) )) unless !File.exists?( default_config_file ) - options end @@ -314,10 +311,21 @@ class PfttOptions < OptionsHash ) do $force_deploy = true end + + opts.on( + '--at' + ) do + $auto_triage = true + end + + opts.on( + '--p' + ) do + # LATER xhprof profiling support + end opts.on( - '--debug', - 'Uses WinDbg or GDB to debug PHP while its running tests (usually only used with func_part).' + '--d' ) do $debug_test = true end @@ -347,7 +355,7 @@ class PfttOptions < OptionsHash '--clear', 'Clear the loaded configuration entirely. Helpful for scenarios in which you do not want to use the default configuration, or want to clear out a particular setting.' ) do - clear # TODO + clear end opts.on( @@ -411,8 +419,7 @@ class PfttOptions < OptionsHash end opts.on( - '--i[int[eractive]]', - 'Interactive Mode, breaks intto a debug prompt when different test output is encountered.' + '--i[int[eractive]]' ) do $interactive_mode = true end @@ -597,10 +604,11 @@ CONFIG = PfttOptions.parse(ARGV) # set up our basic test bench factors $hosts = (Host::Array.new.load(CONFIG[:host,:path].convert_path)).filter(CONFIG[:host,:filters]) -$hosts.push(Host::Local.new()) # TODO +$hosts.push(Host::Remote::Ssh.new(:address=>'127.0.0.1', :username=>'administrator', :password=>'password01!'))#Host::Local.new()) # TODO require 'typed-array' $phps = PhpBuild.get_set(CONFIG[:php,:dir].convert_path||'').filter(CONFIG[:php,:filters]) -$middlewares = [Middleware::Cli] # TODO Middleware::All#.filter([])# TODO CONFIG[:middleware,:filters]) +$middlewares = [Middleware::Cli]#, + #Middleware::Http::IIS::FastCgi::Base]#, Middleware::Http::Apache::ModPhp::Base] # TODO Middleware::All#.filter([])# TODO CONFIG[:middleware,:filters]) # LATER? what about NFSv3 support (which ships with Windows 7<) (not NFSv4) $scenarios = {} @@ -711,12 +719,20 @@ if __FILE__ == $0 puts 'PFTT: host_config: ensuring 7zip installed...' win_install(host, "7z920.msi", '/Q') - # TODO install elevate - # TODO install wcat + # install elevate + if win_install_installed?('elevate') + win_install_copy(host, 'Elevate') + + # doesn't have an MSI, just a batch script + host.exec!('Elevate\\InstallAllPowerToys.cmd') + end + + # install wcat (hopefully this is a x64 host) + win_install(host, 'wcat.amd64.msi', '/Q') puts 'PFTT: host_config: ensuring IIS installed...' - unless host.exits?('%SYSTEMDIR%/system32/inetsrv/appcmd.exe') + unless host.exists?('%SYSTEMDIR%/system32/inetsrv/appcmd.exe') # IIS is not installed. try to install it host.deploy('scripts/IIS/iis_unattend.xml', host.systemdrive+'/iis_unattend.xml') @@ -733,12 +749,12 @@ if __FILE__ == $0 puts 'PFTT: host_config: ensuring Apache installed...' linux_install(host, "apache-httpd") end - # - # TODO install svn + # install SVN + install(host, 'svn', 'svn', 'Setup-Subversion-1.6.17.msi', '/Q') end def update_config puts 'PFTT: updating configuration from server...' - # TODO update_config + # LATER update_config end def scenario_config # LATER scenario config @@ -766,7 +782,7 @@ if __FILE__ == $0 f.close exit - elsif CONFIG[:action] == 'func_full' or CONFIG[:action] == 'func_part' + elsif CONFIG[:action] == 'func_full' or CONFIG[:action] == 'func_part' or CONFIG[:action] == 'perf' # func_full - run all PHPTs (always deploy php build and test cases too) # func_part - run selected PHPTs (deploy php build and/or test cases if modified) @@ -795,10 +811,12 @@ if __FILE__ == $0 end # - $testcases = CONFIG.selected_tests() + unless CONFIG[:action] == 'perf' + $testcases = CONFIG.selected_tests() + end # - # + # add more threads to keep track of more hosts, but limit the size $thread_pool_size = $thread_pool_size * $hosts.length if $thread_pool_size > 60 $thread_pool_size = 60 @@ -832,13 +850,17 @@ if __FILE__ == $0 host_config end - test_bench = TestBench::Phpt.new($phps, $hosts, $middlewares[0], $scenarios) # TODO + if CONFIG[:action] == 'perf' + test_bench = TestBench::Wcat.new($phps, $hosts, $middlewares[0], $scenarios) + else + test_bench = TestBench::Phpt.new($phps, $hosts, $middlewares[0], $scenarios) # TODO + end # TODO print telemetry folder before starting (so user can read telemetry in real time) # finally do the testing... # iterate over all hosts, middlewares, etc..., running all tests for each - test_ctx = test_bench.iterate( $phps, $hosts, $middlewares, $scenarios, $testcases ) # TODO + test_ctx = test_bench.iterate( $phps, $hosts, $middlewares, $scenarios, $testcases ) ensure # ensure hosts are unlocked @@ -870,7 +892,12 @@ if __FILE__ == $0 # for func_part user shouldn't have to take time navigating through a redudant report # to get to the telemetry they need - report = Report::Run::ByHost::Func.new() + if CONFIG[:action] == 'perf' + report == Report::Run::ByHost::Perf.new() + else + report = Report::Run::ByHost::Func.new() + end + report.text_print() @@ -898,11 +925,6 @@ if __FILE__ == $0 exit elsif CONFIG[:action] == 'cs' scenario_config - elsif CONFIG[:action] == 'perf' - # TODO implement performance testing - # - puts "PFTT: error: performance testing not implemented" - exit(1) elsif CONFIG[:action] == 'stress' # LATER implement stress testing # LATER run ui test periodically during stress test diff --git a/PFTT/lib/diff.rb b/PFTT/lib/diff.rb index aaaa5c1..c44fe68 100644 --- a/PFTT/lib/diff.rb +++ b/PFTT/lib/diff.rb @@ -150,6 +150,7 @@ module Diff puts '<Enter> - skip change' puts ' s - skip line' puts ' S - skip file' + puts ' f - runs tests non-interactively, then prompt to re-run' # TODO puts ' r - replace expect with regex to match actual' puts ' R - replace all in file' puts ' A - replace all in test case set' @@ -176,7 +177,7 @@ module Diff # This method gets replaced by sub-classes and is the part that does the actual # comparrisons. def _compare_line( expectation, result ) - expectation == result or expectation.chomp == result.chomp + expectation == result or expectation.rstrip.chomp == result.rstrip.chomp end def _get_diff( expectation, result ) @@ -504,7 +505,7 @@ module Diff class Exact < Base def _compare_line( expectation, result ) - expectation == result or expectation.chomp == result.chomp + expectation == result or expectation.rstrip.chomp == result.rstrip.chomp end end @@ -512,7 +513,7 @@ module Diff def _compare_line( expectation, result ) #puts %Q{compare: #{expectation.inspect} to #{result.inspect}} r = Regexp.new(%Q{\\A#{expectation}\\Z}) # TODO - r.match(result) or r.match(result.chomp) + r.match(result) or r.match(result.rstrip.chomp) end end @@ -546,13 +547,13 @@ module Diff def _compare_line( expectation, result ) if expectation == nil || result == nil return false - elsif expectation == result or expectation.chomp == result.chomp + elsif expectation == result or expectation.rstrip.chomp == result.rstrip.chomp return true #elsif result.include?('string(70)') # return true end #puts expectation - rex = Regexp.escape(expectation.chomp) + rex = Regexp.escape(expectation.rstrip.chomp) # arrange the patterns in longest-to shortest and apply them. # the order matters because %string% must be replaced before %s. @@ -573,7 +574,7 @@ module Diff #puts result r = Regexp.new(rex) - r.match(result) != nil or r.match(result.chomp) != nil + r.match(result) != nil or r.match(result.rstrip.chomp) != nil #if !b and result.include?('string(70)') # exit #end diff --git a/PFTT/lib/host.rb b/PFTT/lib/host.rb index eefe292..821725f 100644 --- a/PFTT/lib/host.rb +++ b/PFTT/lib/host.rb @@ -15,13 +15,24 @@ module Host include TestBenchFactor include PhpIni::Inheritable + def systemroot + if posix? + return '/' + elsif @_systemroot + return @_systemroot + else + @_systemroot = unquote_line!('echo %SYSTEMROOT%') + return @_systemroot + end + end + def systemdrive if posix? return '/' elsif @_systemdrive return @_systemdrive else - @_systemdrive = line!('echo %SYSTEMDRIVE%').gsub(/\"/, '') + @_systemdrive = unquote_line!('echo %SYSTEMDRIVE%') return @_systemdrive end end @@ -42,6 +53,7 @@ module Host if posix? @_osname = line!('uname -a') else + # LATER use systeminfo and parse it locally (otherwise windows has to run systeminfo 2 or 3 times, and its slow) osname = line_prefix!('OS Name:', 'systeminfo | findstr /B /C:"OS Name"') osname += ' ' osname += line_prefix!('System Type:', 'systeminfo | findstr /B /C:"System Type"') # x86 or x64 @@ -148,13 +160,17 @@ module Host end return exec!(cmdline) end - + # executes command using cmd! returning the first line of output (STDOUT) from the command, # with the new line character(s) chomped off def line! cmdline cmd!(cmdline)[0].chomp end + def unquote_line! cmdline + line!(cmdline).gsub(/\"/, '') + end + def windows? # avoids having to check for c:\windows|c:\winnt if we've already found /usr/local @is_windows ||= ( !posix? and ( exist? "C:\\Windows" or exist? "C:\\WinNT" ) ) @@ -195,7 +211,11 @@ module Host end def popd - cd(@dir_stack.pop, {:no_clear=>true}) + popped = @dir_stack.pop + if popped + # if nil, fail silently + cd(popped, {:no_clear=>true}) + end end def peekd @@ -280,14 +300,31 @@ module Host end end + def name + unless @_name + # find a name that other hosts on the network will use to reference localhost + if windows? + @_name = line!('echo %COMPUTERNAME%') + else + @_name = line!('echo $HOSTNAME') + end + end + @_name + end + protected + attr_accessor :_name, :_osname, :_systeminfo, :_systemdrive, :_systemroot, :posix, :is_windows + def clone(clone) -# TODO clone._osname = @_osname -# clone._systeminfo = @_systeminfo -# clone._systemdrive = @_systemdrive -# clone.posix = @posix -# clone.is_windows = @is_windows + clone._osname = @_osname + clone._systemroot = @_systemroot + clone._systeminfo = @_systeminfo + clone._systemdrive = @_systemdrive + clone.posix = @posix + clone.is_windows = @is_windows + clone._name = @_name + clone end end # class Base diff --git a/PFTT/lib/host/local.rb b/PFTT/lib/host/local.rb index 703a07f..2359567 100644 --- a/PFTT/lib/host/local.rb +++ b/PFTT/lib/host/local.rb @@ -4,10 +4,6 @@ module Host class Local < Base instantiable 'local' - def name - 'localhost' - end - def address '127.0.0.1' end @@ -15,8 +11,7 @@ module Host def clone clone = Host::Local.new() # clone may have a different @dir_stack, so it really must be cloned - super(clone) - clone + super(clone) end def close @@ -151,7 +146,7 @@ module Host make_absolute! spec Dir.glob spec, &block end - + protected def _delete path @@ -163,5 +158,6 @@ module Host make_absolute! path Dir.mkdir path end - end + + end # class Local end \ No newline at end of file diff --git a/PFTT/lib/host/remote/ssh.rb b/PFTT/lib/host/remote/ssh.rb index e0a97c0..f1d4c9d 100644 --- a/PFTT/lib/host/remote/ssh.rb +++ b/PFTT/lib/host/remote/ssh.rb @@ -13,11 +13,17 @@ module Host instantiable 'ssh' def name - @credentials[:host_name] + n = @credentials[:host_name] + if n.include?('.') + # name is an address, try getting the host's name for itself + return super + else + return n + end end def address - name + @credentials[:host_name] end def close @@ -42,13 +48,12 @@ module Host end def clone - clone = Host::Remote::Ssh.new({ - :address => @credentials[:host_name], - :user => @credentials[:user], - :password => @credentials[:password] - }) + clone = Host::Remote::Ssh.new( + :address => @credentials[:host_name], + :username => @credentials[:user], + :password => @credentials[:password] + ) super(clone) - clone end def alive? @@ -71,7 +76,7 @@ module Host def exec command, opts={} @cwd = nil # clear cwd cache - if opts.has_key(:chdir) + if opts.has_key?(:chdir) restore_cd = cwd cd(opts[:chdir]) else @@ -142,11 +147,11 @@ module Host def cwd @cwd ||= case when posix? then line!('pwd') - else line!("ECHO %CD%").gsub(/\"/, '') + else unquote_line!("ECHO %CD%") end end - def cd path, hsh + def cd path, hsh={} make_absolute! path if not path # popd may have been called when @dir_stack empty @@ -269,12 +274,20 @@ module Host end def ssh - @ssh ||= Net::SSH.start( nil, nil, @credentials ) + @ssh ||= Net::SSH.start( nil, nil, { + :host_name => @credentials[:host_name], + :user => @credentials[:user], + :password => @credentials[:password] + }) @ssh end def sftp - @sftp ||= Net::SFTP.start( nil, nil, @credentials ) + @sftp ||= Net::SFTP.start( nil, nil, { + :host_name => @credentials[:host_name], + :user => @credentials[:user], + :password => @credentials[:password] + }) @sftp end end diff --git a/PFTT/lib/middleware/cli.rb b/PFTT/lib/middleware/cli.rb index d66eb15..8a3e2ed 100644 --- a/PFTT/lib/middleware/cli.rb +++ b/PFTT/lib/middleware/cli.rb @@ -4,6 +4,10 @@ module Middleware instantiable property :interface => 'cli' + def self.mw_name + 'Command-Line' + end + def clone clone = Middleware::Cli.new(@host.clone, @php_build, @scenarios) clone.deployed_php = @deployed_php diff --git a/PFTT/lib/middleware/http.rb b/PFTT/lib/middleware/http.rb index 9b1626d..69a2900 100644 --- a/PFTT/lib/middleware/http.rb +++ b/PFTT/lib/middleware/http.rb @@ -5,9 +5,10 @@ require "uri" module Middleware module Http class HttpBase < Base - # TODO property :interface => 'http' + property :interface => 'http' + def translate_path(deployed_script) # TODO - 'http://127.0.0.1:8080/'+deployed_script.gsub('C:/inetpub/wwwroot', '').gsub('C:/Program Files (x86)/Apache Software Foundation/Apache2.2/htdocs/', '') + 'http://127.0.0.1:8080/'+deployed_script.gsub('C:\\Users\v-mafick\Desktop\sf\workspace\SSHD','').gsub('C:/inetpub/wwwroot', '').gsub('C:/Program Files (x86)/Apache Software Foundation/Apache2.2/htdocs/', '') end def execute_php_script deployed_script, test, script_type, scenarios diff --git a/PFTT/lib/middleware/http/apache.rb b/PFTT/lib/middleware/http/apache.rb index 32e3095..7af9c4f 100644 --- a/PFTT/lib/middleware/http/apache.rb +++ b/PFTT/lib/middleware/http/apache.rb @@ -4,9 +4,10 @@ module Middleware class ApacheBase < HttpBase def root r=nil - if r - return r - elsif @apache_root +# TODO trim /htdocs/ if r +# return r +# els + if @apache_root return @apache_root elsif @host.posix? return '/' diff --git a/PFTT/lib/middleware/http/apache/mod_php.rb b/PFTT/lib/middleware/http/apache/mod_php.rb index ceff613..60eabb3 100644 --- a/PFTT/lib/middleware/http/apache/mod_php.rb +++ b/PFTT/lib/middleware/http/apache/mod_php.rb @@ -1,9 +1,9 @@ -# TODO implement APC and WinCache support using Middleware module Middleware module Http module Apache - class ModPhp < ApacheBase + module ModPhp + class Base < ApacheBase requirement :threadsafe => true instantiable @@ -13,8 +13,12 @@ module Middleware self end + def mw_name + 'Apache-ModPHP' + end + def clone - clone = Middleware::Http::Apache::ModPhp.new(@host.clone, @php_build, @scenarios) + clone = Middleware::Http::Apache::ModPhp::Base.new(@host.clone, @php_build, @scenarios) clone.deployed_php = @deployed_php clone end @@ -37,7 +41,7 @@ module Middleware def start! if @host.windows? # make sure apache is installed as a windows service (for 'net start') - @host.exec! ((apache_root+'bin/httpd -k install').convert_path) + @host.exec! ((root+'bin/httpd -k install').convert_path) # makre sure IIS is turned off @host.exec! 'net stop w3svc' @@ -79,6 +83,7 @@ module Middleware applied end end + end end end end \ No newline at end of file diff --git a/PFTT/lib/middleware/http/apache/mod_php/apc.rb b/PFTT/lib/middleware/http/apache/mod_php/apc.rb index e69de29..8f96fe6 100644 --- a/PFTT/lib/middleware/http/apache/mod_php/apc.rb +++ b/PFTT/lib/middleware/http/apache/mod_php/apc.rb @@ -0,0 +1,21 @@ + +module Middleware + module Http + module Apache + module ModPhp + module APC + class Base < Base + def mw_name + 'Apache-ModPHP-APC' + end + def clone + clone = Middleware::Http::Apache::ModPhp::APC::Base.new(@host.clone, @php_build, @scenarios) + clone.deployed_php = @deployed_php + clone + end + end + end + end + end + end +end diff --git a/PFTT/lib/middleware/http/apache/mod_php/apc/igbinary.rb b/PFTT/lib/middleware/http/apache/mod_php/apc/igbinary.rb index e69de29..171a288 100644 --- a/PFTT/lib/middleware/http/apache/mod_php/apc/igbinary.rb +++ b/PFTT/lib/middleware/http/apache/mod_php/apc/igbinary.rb @@ -0,0 +1,21 @@ + +module Middleware + module Http + module Apache + module ModPhp + module APC + class IGBinary < Base + def mw_name + 'Apache-ModPHP-APC-IGBinary' + end + def clone + clone = Middleware::Http::Apache::ModPhp::APC::IGBinary.new(@host.clone, @php_build, @scenarios) + clone.deployed_php = @deployed_php + clone + end + end + end + end + end + end +end diff --git a/PFTT/lib/middleware/http/iis.rb b/PFTT/lib/middleware/http/iis.rb index cc81658..cc26d4b 100644 --- a/PFTT/lib/middleware/http/iis.rb +++ b/PFTT/lib/middleware/http/iis.rb @@ -1,8 +1,8 @@ module Middleware module Http module IIS - # TODO requirement :platform => :windows class IisBase < HttpBase + requirement :platform => :windows def initialize(*args) super(*args) @@ -19,7 +19,7 @@ module Middleware # root(r) goes through T:/ or other mounted network drive (SMB) # while @host goes through SSH and executes the operation locally on the remote computer def appcmd args - @host.exec! "%SYSTEMROOT%/System32/inetsrv/appcmd #{args}" + @host.exec!(@host.systemroot+"/System32/inetsrv/appcmd #{args}") end def start! diff --git a/PFTT/lib/middleware/http/iis/fast_cgi.rb b/PFTT/lib/middleware/http/iis/fast_cgi.rb index 0525d9e..0dd8523 100644 --- a/PFTT/lib/middleware/http/iis/fast_cgi.rb +++ b/PFTT/lib/middleware/http/iis/fast_cgi.rb @@ -2,10 +2,15 @@ module Middleware module Http module IIS - class FastCgi < IisBase + module FastCgi + class Base < IisBase + + def mw_name + 'IIS-FastCGI' + end def clone - clone = Middleware::Http::IIS::FastCgi.new(@host.clone, @php_build, @scenarios) + clone = Middleware::Http::IIS::FastCgi::Base.new(@host.clone, @php_build, @scenarios) clone.deployed_php = @deployed_php clone end @@ -36,6 +41,7 @@ module Middleware INI end + end end end end diff --git a/PFTT/lib/middleware/http/iis/fast_cgi/wincache.rb b/PFTT/lib/middleware/http/iis/fast_cgi/wincache.rb index e69de29..3f81f40 100644 --- a/PFTT/lib/middleware/http/iis/fast_cgi/wincache.rb +++ b/PFTT/lib/middleware/http/iis/fast_cgi/wincache.rb @@ -0,0 +1,19 @@ + +module Middleware + module Http + module IIS + module FastCgi + class Wincache < Base + def mw_name + 'IIS-FastCGI-WinCache' + end + def clone + clone = Middleware::Http::IIS::FastCgi::Wincache.new(@host.clone, @php_build, @scenarios) + clone.deployed_php = @deployed_php + clone + end + end + end + end + end +end diff --git a/PFTT/lib/pftt_client.rb b/PFTT/lib/pftt_client.rb index 08bb8e1..1127f49 100644 --- a/PFTT/lib/pftt_client.rb +++ b/PFTT/lib/pftt_client.rb @@ -48,20 +48,19 @@ class PfttClient def release(host_name, lock_id) end def view - # platform = host_info[:os].to_s.sub('Windows', 'Win') return [ - {:host_name=>'OI1-PHP-WDW-10', :status=>:ready, :os_short=>'Win 2003r2 SP0', :os=>'Windows 2003r2 SP0', :arch=>'x64', :ip_address=>'10.200.30.11'}, - {:host_name=>'OI1-PHP-WDW-25', :status=>:locked, :os_short=>'Win Vista SP2', :os=>'Windows Vista SP2', :arch=>'x86', :ip_address=>'10.200.30.12'}, - {:host_name=>'OI1-PHP-WDW-29', :status=>:ready, :os_short=>'Win 8 Client M3', :os=>'Windows 8 Client M3', :arch=>'x86', :ip_address=>'10.200.30.13'}, - {:host_name=>'PBS-0', :status=>:ready, :os_short=>'Gentoo Linux', :os=>'Gentoo Linux', :arch=>'x64', :ip_address=>'10.200.30.14'}, - {:host_name=>'PBS-1', :status=>:locked, :os_short=>'Gentoo Linux', :os=>'Gentoo Linux', :arch=>'x64', :ip_address=>'10.200.30.15'}, - {:host_name=>'PBS-2', :status=>:ready, :os_short=>'Gentoo Linux', :os=>'Gentoo Linux', :arch=>'x64', :ip_address=>'10.200.30.16'}, - {:host_name=>'AZ-WEB-PHP-0', :status=>:ready, :os_short=>'Azure Web 2008', :os=>'Azure Web 2008', :arch=>'x64', :ip_address=>'157.60.40.11'}, - {:host_name=>'AZ-VM-PHP-0', :status=>:ready, :os_short=>'Azure VM 2008', :os=>'Azure VM 2008', :arch=>'x64', :ip_address=>'157.60.40.12'}, - {:host_name=>'AZ-WKR-PHP-0', :status=>:ready, :os_short=>'Azure Worker 2008', :os=>'Azure Worker 2008', :arch=>'x64', :ip_address=>'157.60.40.13'}, - {:host_name=>'AZ-WEB-PHP-1', :status=>:locked, :os_short=>'Azure Web 2008r2', :os=>'Azure Web 2008r2', :arch=>'x64', :ip_address=>'157.60.40.14'}, - {:host_name=>'AZ-VM-PHP-1', :status=>:ready, :os_short=>'Azure VM 2008r2', :os=>'Azure VM 2008r2', :arch=>'x64', :ip_address=>'157.60.40.15'}, - {:host_name=>'AZ-WKR-PHP-1', :status=>:ready, :os_short=>'Azure Worker 2008r2', :os=>'Azure Worker 2008r2', :arch=>'x64', :ip_address=>'157.60.40.16'} + {:host_name=>'OI1-PHP-WDW-10', :status=>:ready, :os=>'Windows 2003r2 SP0', :arch=>'x64', :ip_address=>'10.200.30.11'}, + {:host_name=>'OI1-PHP-WDW-25', :status=>:locked, :os=>'Windows Vista SP2', :arch=>'x86', :ip_address=>'10.200.30.12'}, + {:host_name=>'OI1-PHP-WDW-29', :status=>:ready, :os=>'Windows 8 Client M3', :arch=>'x86', :ip_address=>'10.200.30.13'}, + {:host_name=>'PBS-0', :status=>:ready, :os=>'Gentoo Linux', :arch=>'x64', :ip_address=>'10.200.30.14'}, + {:host_name=>'PBS-1', :status=>:locked, :os=>'Gentoo Linux', :arch=>'x64', :ip_address=>'10.200.30.15'}, + {:host_name=>'PBS-2', :status=>:ready, :os=>'Gentoo Linux', :arch=>'x64', :ip_address=>'10.200.30.16'}, + {:host_name=>'AZ-WEB-PHP-0', :status=>:ready, :os=>'Azure Web 2008', :arch=>'x64', :ip_address=>'157.60.40.11'}, + {:host_name=>'AZ-VM-PHP-0', :status=>:ready, :os=>'Azure VM 2008', :arch=>'x64', :ip_address=>'157.60.40.12'}, + {:host_name=>'AZ-WKR-PHP-0', :status=>:ready, :os=>'Azure Worker 2008', :arch=>'x64', :ip_address=>'157.60.40.13'}, + {:host_name=>'AZ-WEB-PHP-1', :status=>:locked, :os=>'Azure Web 2008r2', :arch=>'x64', :ip_address=>'157.60.40.14'}, + {:host_name=>'AZ-VM-PHP-1', :status=>:ready, :os=>'Azure VM 2008r2', :arch=>'x64', :ip_address=>'157.60.40.15'}, + {:host_name=>'AZ-WKR-PHP-1', :status=>:ready, :os=>'Azure Worker 2008r2', :arch=>'x64', :ip_address=>'157.60.40.16'} ] end end diff --git a/PFTT/lib/php_build.rb b/PFTT/lib/php_build.rb index 8910313..645748b 100644 --- a/PFTT/lib/php_build.rb +++ b/PFTT/lib/php_build.rb @@ -85,11 +85,14 @@ class PhpBuild end if base_name == 'PHP_5_4' - property :php_branch => '5_4' + property :php_branch => 'PHP_5_4' property :php_version_major => '5' + property :php_version_minor => '4' elsif base_name == 'PHP_5_3' - property :php_branch => '5_3' + property :php_branch => 'PHP_5_3' property :php_version_major => '5' + property :php_version_minor => '3' + # LATER fix branch parsing support so it'll work for any version(not just 5_4 and 5_3) else raise 'Unsupported PHP Build Type(Info Format:)' end @@ -117,8 +120,10 @@ class PhpBuild branchinfo = parts.select{|i| i =~ /(?:[0-9]\.[0-9]+|trunk)/ }.first - property :php_branch => branchinfo.split('.').first(2).join('.') - property :php_version_major => branchinfo.split('.').first.to_i + bs = branchinfo.split('.') + property :php_branch => bs.first(2).join('_') + property :php_version_major => bs.first.to_i + property :php_version_minor => bs[1].to_i property :threadsafe => parts.select{|i| i == 'nts' }.empty? property :revision => (parts.select{|i| i =~ /r[0-9]+/ }).first diff --git a/PFTT/lib/phpt_test_result.rb b/PFTT/lib/phpt_test_result.rb index 20a61d6..963b8de 100644 --- a/PFTT/lib/phpt_test_result.rb +++ b/PFTT/lib/phpt_test_result.rb @@ -14,8 +14,7 @@ module PhptTestResult %Q{[#{status.to_s.upcase}] #{@test_bench} #{@test_case.relative_path}} end - def save - base_path = test_bench.telemetry_folder + def save(base_path) FileUtils.mkdir_p base_path File.open( File.join( base_path, %Q{#{status.to_s.upcase}.list} ), 'a' ) do |file| @@ -102,7 +101,7 @@ module PhptTestResult end class Meaningful < Base - def initialize *args, result_str + def initialize *args, php, result_str super *args @result_str = result_str @@ -116,7 +115,7 @@ module PhptTestResult # TODO :expectregex must be evaluated as a regular expression, will this do that?? when :expectregex then Diff::Exact when :expectf - case @test_bench.php.properties[:php_version_major] + case php.properties[:php_version_major] when 5 then Diff::Formatted::Php5 when 6 then Diff::Formatted::Php6 else Diff::Formatted @@ -196,14 +195,20 @@ module PhptTestResult end class Array < TypedArray(PhptTestResult::Base) + attr_accessor :telemetry_folder, :windows_ini, :posix_ini, :run_time, :exceptions def initialize - @telemetry_url = '' - @run_id = '' + #ctx_id # ties each PhptTestResult::Array together with a TestBenchRunContext + # + #@telemetry_url = '' + @telemetry_folder = '' + #@run_id = '' @windows_ini = '' @posix_ini = '' - @start_time - @end_time + @run_time = 0 + @exceptions = [] + #@start_time + #@end_time end @@ -305,7 +310,7 @@ module PhptTestResult exts = {} self.each do |result| - if show==:all or result.status==:show + if show==:all or (show==:run and (result.status==:pass or result.status==:fail or result.status==:works or result.status==:pass)) or result.status==show exts[result.test_case.ext_name] = true end end diff --git a/PFTT/lib/report/comparison/by_platform/func_release_comparison.rb b/PFTT/lib/report/comparison/by_platform/func_release_comparison.rb index a97b2ed..92205a1 100644 --- a/PFTT/lib/report/comparison/by_platform/func_release_comparison.rb +++ b/PFTT/lib/report/comparison/by_platform/func_release_comparison.rb @@ -16,9 +16,7 @@ module Report def write_end "<p>* - tests missing required sections (test is not runnable)</p><p>** - tests with unsupported sections (test is not runnable)</p></body></html>" end - - # TODO if consecutive rows have same results, merge them into one row and put both SKUs (including arch) in the SKU column - # implement in by_platform.rb + def write_table(cm) # colors to use bgcolor_os = '#f2f2f2' diff --git a/PFTT/lib/report/comparison/by_platform/perf_release_comparison.rb b/PFTT/lib/report/comparison/by_platform/perf_release_comparison.rb index 309beb6..43ab751 100644 --- a/PFTT/lib/report/comparison/by_platform/perf_release_comparison.rb +++ b/PFTT/lib/report/comparison/by_platform/perf_release_comparison.rb @@ -18,41 +18,45 @@ module Report end def write_app_detail_table(app, cm) - # TODO share row numbering with write_comparison_table cm.add_row({:text=>app, :colspan=>8}) cm.add_row('', 'OS', 'vrtu', 'tps', 'kcpt', 'bpt', 'cpu', 'err') return cm end - # TODO if consecutive rows have same results, merge them into one row and put both SKUs (including arch) in the SKU column - # implement in by_platform.rb - def write_comparison_table(app, cm) - - row_num = 0 # TODO use same numbering for all rows - - [['context_1', 'context_2'], ['context_1', 'context_3']].each{|ctx_set| + def write_comparison_table(app, cm, r) + r[TypedToken::ArrayToken::ScenarioSet].each{|scn_set| cm.add_row( {:text=>(cm.html?)?'<strong>Scenarios:</strong>':'Scenarios:', :colspan=>2}, - {:text=>ctx_set.inspect.to_s, :colspan=>20} + {:text=>scn_set.inspect.to_s, :colspan=>20} ) cm.add_row({:text=>'', :colspan=>4}, {:text=>'IIS', :colspan=>6}, {:text=>'Apache', :colspan=>9}) cm.add_row({:text=>'', :colspan=>4}, {:text=>'Load Agents', :colspan=>2}, {:text=>'No WinCache', :colspan=>3}, {:text=>'WinCache', :colspan=>3}, {:text=>'No APC', :colspan=>3}, {:text=>'APC', :colspan=>3}, {:text=>'APC w/ IGBinary', :colspan=>3}) cm.add_row('', 'OS', 'Physical', 'Virtual', 'Base', 'Test', 'Gain', 'Base', 'Test', 'Gain', 'Base', 'Test', 'Gain', 'Base', 'Test', 'Gain') - cm.add_row('1', 'Win 2003 x86 SP2', '2', '8', '50', '100', '50%', '100', '50', '50%', '50', '100', '50%', '100', '50', '50%') + windows_ini, posix_ini = nil + r[scn_set, TypedToken::StringToken::OSName].each{|results| + # TODO + cm.add_row( + {:row_number=>true}, + os_short_name(results.os), + results.physical_client, + results.virtual_client, + '50', '100', '50%', '100', '50', '50%', '50', '100', '50%', '100', '50', '50%') + + windows_ini = results.windows_ini + posix_ini = results.posix_ini + } cm.add_row( {:text=>(cm.html?)?'<strong>Windows INI:</strong>':'INI:', :colspan=>2}, - # TODO show INI - {:text=>ctx_set.inspect.to_s, :colspan=>20} + {:text=>windows_ini.to_s, :colspan=>20} ) cm.add_row( - {:text=>(cm.html?)?'<strong>Linux INI:</strong>':'INI:', :colspan=>2}, - # TODO show INI - {:text=>ctx_set.inspect.to_s, :colspan=>20} + {:text=>(cm.html?)?'<strong>POSIX INI:</strong>':'INI:', :colspan=>2}, + {:text=>posix_ini.to_s, :colspan=>20} ) @@ -64,13 +68,13 @@ module Report def write_tables(cm) html_txt = '' - apps = ['Hello World', 'Typo3', 'Drupal', 'MediaWiki', 'Wordpress', 'Joomla'] - apps.each{|app| - html_txt += write_comparison_table(app, cm.clone).render() + apps = @results_ctx[TypedToken::StringToken::AppName]#['Hello World', 'Typo3', 'Drupal', 'MediaWiki', 'Wordpress', 'Joomla'] + apps.keys.each{|app| + html_txt += write_comparison_table(app, cm.clone, apps).render() } - apps.each{|app| - html_txt += write_app_detail_table(app, cm.clone).render() + apps.keys.each{|app| + html_txt += write_app_detail_table(app, cm.clone, apps).render() } return html_txt end diff --git a/PFTT/lib/report/network.rb b/PFTT/lib/report/network.rb index 072a343..df9d51b 100644 --- a/PFTT/lib/report/network.rb +++ b/PFTT/lib/report/network.rb @@ -1,4 +1,6 @@ +require 'util.rb' # for os_short_name + module Report class Network < Base def write_text @@ -49,7 +51,7 @@ module Report status = 'LOCKED' # make this stand out end - cm.add_row({:row_number=>true}, host_info[:host_name], status, host_info[:os_short], host_info[:arch], host_info[:ip_address]) + cm.add_row({:row_number=>true}, host_info[:host_name], status, os_short_name(host_info[:os]), host_info[:arch], host_info[:ip_address]) end end diff --git a/PFTT/lib/report/run/per_host.rb b/PFTT/lib/report/run/per_host.rb index d5f72bb..8d712f9 100644 --- a/PFTT/lib/report/run/per_host.rb +++ b/PFTT/lib/report/run/per_host.rb @@ -6,6 +6,11 @@ module Report attr_reader :host def initialize(host) @host = host + + # cache system info now + # (this is probably not in a synchronized state now, but will probably be when the report + # is written. so it'll avoid locking all threads to do this now) + @host.systeminfo end end end diff --git a/PFTT/lib/report/run/per_host/per_build/per_middleware.rb b/PFTT/lib/report/run/per_host/per_build/per_middleware.rb index 87c8299..ec8799b 100644 --- a/PFTT/lib/report/run/per_host/per_build/per_middleware.rb +++ b/PFTT/lib/report/run/per_host/per_build/per_middleware.rb @@ -5,10 +5,11 @@ module Report module PerBuild module PerMiddleware class Base < Base - attr_reader :middleware - def initialize(host, php_build, middleware) + attr_reader :middleware, :results + def initialize(host, php_build, middleware, results) super(host, php_build) @middleware = middleware + @results = results end end end diff --git a/PFTT/lib/report/run/per_host/per_build/per_middleware/func.rb b/PFTT/lib/report/run/per_host/per_build/per_middleware/func.rb index ba0ce62..3932766 100644 --- a/PFTT/lib/report/run/per_host/per_build/per_middleware/func.rb +++ b/PFTT/lib/report/run/per_host/per_build/per_middleware/func.rb @@ -1,4 +1,6 @@ +require 'util.rb' # for os_short_name + module Report module Run module PerHost @@ -6,24 +8,26 @@ module Report module PerMiddleware class Func < Base def write_text - r = PhptTestResult::Array.new() # TODO + r = @results # use 'r' because its a shorter name str = "\r\n" str += " == Host Run Completed == \r\n" str += "\r\n" - str += "Host: "+(@host.name||@host.address)+"\r\n" - str += "OS(actual): "+@host.osname+"\r\n" - str += ("PHP Build: branch #{@php_build[:php_branch]} major #{@php_build[:php_version_major]} minor #{@php_build[:php_version_minor]} revision #{@php_build[:revision]} SDK "+@php_build.sdk_info.values.to_s+" "+((@host.windows?)?((@php_build.sdk_info[:major]==6)?'':' (Only SDK 6 supported! may cause different output)'):''))+"\r\n" - str += "Middleware: \r\n"# TODO middleware - # TODO puts "Telemetry Folder: "+test_ctx.telemetry_folder + str += "Host: "+@host.name+" ("+@host.address+")\r\n" + str += "OS(actual): "+os_short_name(@host.osname)+"\r\n" + str += ("PHP: branch #{@php_build[:php_branch]} major #{@php_build[:php_version_major]} minor #{@php_build[:php_version_minor]} rev #{@php_build[:revision]} "+(@php_build[:threadsafe]?'TS':'NTS')+" SDK "+@php_build.sdk_info.values.to_s+" "+((@host.windows?)?((@php_build.sdk_info[:major]==6)?'':' (Only SDK 6 supported! may cause different output)'):''))+"\r\n" + str += "Middleware: #{@middleware.mw_name}\r\n" + if r.telemetry_folder + str += "Telemetry Folder: #{r.telemetry_folder}\r\n" + elsif r.telemetry_url + str += "Telemetry URL: #{r.telemetry_url}\r\n" + end - # TODO list number of tests that failed without modification too! - str += "\r\n" str += " === Host Summary === \r\n" str += "\r\n" - # TODO puts ' RUN TIME: '+run_time.to_s+' seconds' + str += " RUN TIME: #{r.run_time.to_s} seconds\r\n" str += ' EXTENSIONS: all '+r.ext.length.to_s+' run '+r.ext(:run).length.to_s+' skipped '+r.ext(:skip).length.to_s+"\r\n" str += "\r\n" str += ' TOTAL: '+r.total.to_s+"\r\n" @@ -34,6 +38,7 @@ module Report str += ' XSKIP: '+r.xskip.to_s+"\r\n" str += ' BORK: '+r.bork.to_s+( r.bork > 0 ? ' PHPT test(s) missing required section(s)': '')+"\r\n" str += ' UNSUPPORTED: '+r.unsupported.to_s+( r.unsupported > 0 ? ' PHPT test(s) using unsupported section(s)': '')+"\r\n" + str += ' PFTT Errors: '+r.exceptions.length.to_s+"\r\n" str += ' PASS: '+r.pass.to_s+"\r\n" str += ' PASS RATE: '+r.rate.to_s+'% ('+r.pass.to_s+'/'+r.pass_plus_fail.to_s+')'+"\r\n" str += "\r\n" diff --git a/PFTT/lib/test_bench.rb b/PFTT/lib/test_bench.rb index abb6e81..8bcaec1 100644 --- a/PFTT/lib/test_bench.rb +++ b/PFTT/lib/test_bench.rb @@ -11,56 +11,10 @@ module TestBench @scenarios = scenarios # an array of arrays, each of which contains one type end - attr_reader :php, :host, :middleware, :scenarios - - # - # Test each of the supplied factors to see if they are compatible with the others. - # This is a little tricky, as incompatibility is not necessarily two-way. - # - # See TestBenchFactor::compatible? for conventions on how these are defined. - # - def compatible? - catch(:compatibility) do - [php,host,middleware].permutation(2) do |factor1,factor2| - throw :compatibility, false unless factor1.meets_requirements_of? factor2 - end - throw :compatibility, true - end - end - - attr_reader :php, :host, :middleware - - # pass calls to the TestBench to the associated Middleware with PhpBuild and Host - # as prepended arguments. - # - # this -> test_bench.call_script( ) - # - # becomes -> (test_bench.middleware).call_script( (test_bench.php), (test_bench.host), 'foo.php' ) - # - [ - :install, - :uninstall, - :apply_ini, - :call_script, - ].compact.each do |method_name| - define_method method_name do |*args,&block| - return @middleware.method(method_name).call( *args, &block ) - end # TODO temp - end - - def describe - @description ||= [ - %Q{#{@php.properties[:version]}-#{DateTime.now.strftime('%Y%m%d-%H%M%S')}}, - @middleware.describe, - @host.describe, - @php.describe( :branch, :threadsafety, :compiler ) - ].compact - end - #class << self - def telemetry_folder - 'c:/' # TODO @telemetry_folder = File.join("#{APPROOT}/results", describe()) - end +# def telemetry_folder +# 'c:/' @telemetry_folder = File.join("#{APPROOT}/results", describe()) +# end # TestBench.iterate( php_builds, hosts, middlewares, *args ){|*args| &block } # # First three arguments must be TypedArray instances carrying collections of @@ -75,30 +29,30 @@ module TestBench # def iterate( php, hosts, middleware, scenarios, test_cases ) - result_folder = telemetry_folder - - # LATER, make this thread safe (so its safe for multiple concurrent #iterate calls). currently, database is not thread-safe! - - # create two different database files - # global for all PFTT tests runs - global_db = SQLite3::Database.new($db_file) - # local for this PFTT result folder (its stored in the results folder along with .diff, .skipif, etc... files) - local_db = SQLite3::Database.new(File.join(result_folder, 'results.sqlite')) - - # ensure database tables exist - global_db.execute("CREATE TABLE IF NOT EXISTS runs(run_id INTEGER PRIMARY KEY AUTOINCREMENT, result_folder VARCHAR(255), start_time TIMESTAMP, end_time TIMESTAMP)") - global_db.execute("CREATE TABLE IF NOT EXISTS iteration(iter_id INTEGER PRIMARY KEY AUTOINCREMENT, run_id INTEGER, start_time TIMESTAMP, end_time TIMESTAMP)") - global_db.execute("CREATE TABLE IF NOT EXISTS results(iter_id INTEGER, test_module VARCHAR(15), test_case VARCHAR(30), test_bench VARCHAR(30), result VARCHAR(10))") - global_db.execute("CREATE TABLE IF NOT EXISTS iteration_info(info_id INTEGER PRIMARY KEY AUTOINCREMENT, iter_id INTEGER, info_type VARCHAR(10), info VARCHAR(30))") - global_db.execute("INSERT INTO runs (result_folder, start_time) VALUES (?, datetime('now','localtime'))", result_folder) - global_run_id = global_db.last_insert_row_id - - local_db.execute("CREATE TABLE IF NOT EXISTS runs(run_id INTEGER PRIMARY KEY AUTOINCREMENT, result_folder VARCHAR(255), start_time TIMESTAMP, end_time TIMESTAMP)") - local_db.execute("CREATE TABLE IF NOT EXISTS iteration(iter_id INTEGER PRIMARY KEY AUTOINCREMENT, run_id INTEGER, start_time TIMESTAMP, end_time TIMESTAMP)") - local_db.execute("CREATE TABLE IF NOT EXISTS results(iter_id INTEGER, test_module VARCHAR(15), test_case VARCHAR(30), test_bench VARCHAR(30), result VARCHAR(10))") - local_db.execute("CREATE TABLE IF NOT EXISTS iteration_info(info_id INTEGER PRIMARY KEY AUTOINCREMENT, iter_id INTEGER, info_type VARCHAR(10), info VARCHAR(30))") - local_db.execute("INSERT INTO runs (result_folder, start_time) VALUES (?, datetime('now','localtime'))", result_folder) - local_run_id = local_db.last_insert_row_id +# result_folder = telemetry_folder +# +# # LATER, make this thread safe (so its safe for multiple concurrent #iterate calls). currently, database is not thread-safe! +# +# # create two different database files +# # global for all PFTT tests runs +# global_db = SQLite3::Database.new($db_file) +# # local for this PFTT result folder (its stored in the results folder along with .diff, .skipif, etc... files) +# local_db = SQLite3::Database.new(File.join(result_folder, 'results.sqlite')) +# +# # ensure database tables exist +# global_db.execute("CREATE TABLE IF NOT EXISTS runs(run_id INTEGER PRIMARY KEY AUTOINCREMENT, result_folder VARCHAR(255), start_time TIMESTAMP, end_time TIMESTAMP)") +# global_db.execute("CREATE TABLE IF NOT EXISTS iteration(iter_id INTEGER PRIMARY KEY AUTOINCREMENT, run_id INTEGER, start_time TIMESTAMP, end_time TIMESTAMP)") +# global_db.execute("CREATE TABLE IF NOT EXISTS results(iter_id INTEGER, test_module VARCHAR(15), test_case VARCHAR(30), test_bench VARCHAR(30), result VARCHAR(10))") +# global_db.execute("CREATE TABLE IF NOT EXISTS iteration_info(info_id INTEGER PRIMARY KEY AUTOINCREMENT, iter_id INTEGER, info_type VARCHAR(10), info VARCHAR(30))") +# global_db.execute("INSERT INTO runs (result_folder, start_time) VALUES (?, datetime('now','localtime'))", result_folder) +# global_run_id = global_db.last_insert_row_id +# +# local_db.execute("CREATE TABLE IF NOT EXISTS runs(run_id INTEGER PRIMARY KEY AUTOINCREMENT, result_folder VARCHAR(255), start_time TIMESTAMP, end_time TIMESTAMP)") +# local_db.execute("CREATE TABLE IF NOT EXISTS iteration(iter_id INTEGER PRIMARY KEY AUTOINCREMENT, run_id INTEGER, start_time TIMESTAMP, end_time TIMESTAMP)") +# local_db.execute("CREATE TABLE IF NOT EXISTS results(iter_id INTEGER, test_module VARCHAR(15), test_case VARCHAR(30), test_bench VARCHAR(30), result VARCHAR(10))") +# local_db.execute("CREATE TABLE IF NOT EXISTS iteration_info(info_id INTEGER PRIMARY KEY AUTOINCREMENT, iter_id INTEGER, info_type VARCHAR(10), info VARCHAR(30))") +# local_db.execute("INSERT INTO runs (result_folder, start_time) VALUES (?, datetime('now','localtime'))", result_folder) +# local_run_id = local_db.last_insert_row_id # for each iteration pick one scenario from each key(scenario type) in scenarios # provide a hash to the iteration (ex scenarios[:file_system] should have 1 scenario (not array) within the iteration) @@ -141,12 +95,25 @@ module TestBench test_ctx = TestBenchRunContext.new(self, test_cases.flatten.length) # TODO + + hosts.each do |host| + scenarios.each do |scn_set| + factors = [php, middleware] + factors.shift.product *factors do |php,middleware_spec| + test_ctx.add_legend_label(host, php, middleware_spec) + end + end + end + + # + test_ctx.show_label_legend + + # iterate over all scenario sets (where there is one instance of each scenario type (file system, database)) # each time, iterate over factors(php build, host, middleware) final_test_cases = [] middlewares_uninstall = [] - # TODO once number of results for scenario-host-build-middleware == test_cases.length - # then all test cases for that combination have been run. report the results. + hosts.each do |host| scenarios.each do |scn_set| factors = [php, middleware] @@ -179,7 +146,7 @@ module TestBench } # TODO deploy and teardown of scenarios - test_ctx.results # TODO + test_ctx end def is_started @@ -317,9 +284,12 @@ module TestBench end # end class CombinationGenerator - class TestBenchRunContext + class ResultsContext + end + + class TestBenchRunContext < ResultsContext attr_reader :test_case_len, :semaphore1, :semaphore2, :semaphore3, :semaphore4, :semaphore5, :chunk_replacement - attr_reader :results # TODO temp + def initialize(test_bench, test_case_len) @test_bench = test_bench @test_case_len = test_case_len @@ -330,8 +300,11 @@ module TestBench @semaphore4 = Mutex.new # s_console @semaphore5 = Mutex.new # s_storage - @results = [] + @results = {} @chunk_replacement = {} + + @labels = {} + @labels2 = {} end def console_out(*str) @semaphore4.synchronize { @@ -359,22 +332,140 @@ module TestBench return false end end + def add_legend_label(host, php, middleware) + host_name = host.name + mw_name = middleware.mw_name + version = php[:php_version_minor].to_s + + # + mw_name_i = 0 + while mw_name_i < mw_name.length + host_name_i = 0 + while host_name_i < host_name.length + + name = ( host_name[host_name_i] + mw_name[mw_name_i] + version ).upcase + + unless @labels.has_key?(name) + set_label(host, middleware, host_name, php, mw_name, name) + return name + end + + host_name_i += 1 + end + mw_name_i += 1 + end + # + + # + # fallback + i = @labels.length + while true + name = i.to_s + + unless @labels.has_key?(name) + set_label(host, middleware, host_name, php, mw_name, name) + return name + end + end + + return name + end + def show_label_legend + @semaphore3.synchronize do + puts + puts " Legend Host/PHP/Middleware" + puts + @labels.keys.each do |label| + host_name, php, mw_name = @labels[label] + + puts " #{label} - #{host_name} #{mw_name} #{php.inspect}" + + end + puts + end + end + def legend_label(host, php, middleware) + @labels2[host][middleware.class][php] + end + def add_exception(host, php, middleware, scenarios, ex) + @semaphore3.synchronize do + @results[host]||={} + @results[host][middleware]||={} + results = @results[host][middleware][php] + + unless results + results = @results[host][middleware][php] = PhptTestResult::Array.new() + end + + results.exceptions.push(ex) + end + end def add_result(host, php, middleware, scenarios, result) - # TODO do_finished_host_build_middleware_scenarios = false + results = nil @semaphore3.synchronize do - if @results.length > @test_case_len - raise 'ShouldntHappenError' + @results[host]||={} + @results[host][middleware]||={} + results = @results[host][middleware][php] + + unless results + results = @results[host][middleware][php] = PhptTestResult::Array.new() end - @results.push(result) + if results.length > @test_case_len + raise 'TooManyResultsError' # shouldn't happen + end + + results.push(result) - do_finished_host_build_middleware_scenarios = @results.length == @test_case_len + do_finished_host_build_middleware_scenarios = results.length == @test_case_len end + + telemetry_folder = 'C:/' # TODO + + @semaphore5.synchronize do + # LATER sync saving files in a semaphore unique to each telemetry folder + result.save(telemetry_folder) + end + if do_finished_host_build_middleware_scenarios - @test_bench.finished_host_build_middleware_scenarios(self) + report = @test_bench.finished_host_build_middleware_scenarios(self, telemetry_folder, host, php, middleware, scenarios, results) + + @semaphore4.synchronize do + # write list of scenarios tested + f = File.open(File.join(telemetry_folder, 'scenarios.list'), 'wb') + scenarios.values.each do |scn| + f.puts(scn.scn_name) + end + f.close() + + # write system info too + f = File.open(File.join(telemetry_folder, 'systeminfo.txt'), 'wb') + f.puts(host.systeminfo) + f.close() + + report.text_print() + + # + # + if $interactive_mode + if first_run(host, php, middleware) + if prompt_yesno('PFTT: Re-run and compare the results to first run?') + rerun + end + else + if prompt_yesno('PFTT: Re-run and compare the results to this run?') + set_current_as_first_run(host, php, middleware, self) + rerun + end + end + end + # + # + + end end end def first_run(host, php, middleware) @@ -390,6 +481,16 @@ module TestBench def rerun # TODO repeat run end + + protected + + def set_label(host, middleware, host_name, php, mw_name, name) + @labels[name] = [host_name, php, mw_name] + @labels2[host]||={} + @labels2[host][middleware]||={} + @labels2[host][middleware][php] = name + end + end end # end module TestBench diff --git a/PFTT/lib/test_bench/phpt.rb b/PFTT/lib/test_bench/phpt.rb index 8efe9f1..c369180 100644 --- a/PFTT/lib/test_bench/phpt.rb +++ b/PFTT/lib/test_bench/phpt.rb @@ -18,10 +18,7 @@ module TestBench deployed_location = File.join(@host.cwd, %Q{#{test_case.name}.#{test_case.extension[section]}}) if $force_deploy or not @host.exist?(deployed_location) - @host.open_file(deployed_location,'wb') do |f| - f.write test_case[section] - f.close - end + @host.write(test_case[section], deployed_location) end deployed_location @@ -35,9 +32,7 @@ module TestBench middleware.uninstall(fs_scn.docroot(middleware)) end - # TODO move up to test_bench.rb - def run(test_case_sets, test_ctx)#global_db, local_db, global_iter_id, local_iter_id)#, fs_scn, scenarios) - #results = PhptTestResult::Array.new() # TODO + def run(test_case_sets, test_ctx) # #test_case_sets.map do |test_cases| @@ -53,7 +48,6 @@ module TestBench # TODO scenarios = test_case_sets[0][0][:scenarios][0] - @scen = scenarios # TODO fs_scn = scenarios[:working_file_system] @host = test_case_sets[0][0][:host] @@ -162,11 +156,11 @@ module TestBench test_case.scn_list = scenarios # run the test case! -# TODO begin - do_single_test_case(tmpdir, deployed, skip_if_code_cache, skip_if_result_cache, test_ctx, test_cases, test_case, tmpdir, fs_scn, scenarios) -# rescue -# puts 'PFTT: internal error: do_single_test_case: '+$!.to_s -# end + begin + do_single_test_case(@php, @host, @middleware, tmpdir, deployed, skip_if_code_cache, skip_if_result_cache, test_ctx, test_cases, test_case, tmpdir, fs_scn, scenarios) + rescue + test_ctx.add_exception(@host, @php, @middleware, $!) + end if do_single_thread == 3 test_ctx.semaphore1.synchronize { @@ -216,13 +210,8 @@ module TestBench # end # end def run - def create_per_report() - # TODO @host - Report::Run::PerHost::PerBuild::PerMiddleware::Func.new(@host, @php, @middleware) - end - - def write_ext_file(ext_list, file_name) - ext_file = File.join(telemetry_folder, file_name) # TODO test_ctx.telemetry_folder + def write_ext_file(telemetry_folder, ext_list, file_name) + ext_file = File.join(telemetry_folder, file_name) arw = Util::ArgRewriter.new(ARGV) arw.cmd('func_part') arw.replace('--test-list', file_name) @@ -236,68 +225,32 @@ module TestBench f.close() end - def finished_host_build_middleware_scenarios(test_ctx) - scenarios = @scen # TODO + def finished_host_build_middleware_scenarios(test_ctx, telemetry_folder, host, php, middleware, scenarios, r) + r.telemetry_folder = telemetry_folder # ensure + + report = Report::Run::PerHost::PerBuild::PerMiddleware::Func.new(host, php, middleware, r) # generate a combined INI for all scenarios for both platforms - windows_ini = @middleware.create_ini(scenarios, :windows) - posix_ini = @middleware.create_ini(scenarios, :posix) + r.windows_ini = middleware.create_ini(scenarios, :windows) + r.posix_ini = middleware.create_ini(scenarios, :posix) ini_f = File.open(File.join(telemetry_folder, 'Posix.ini'), 'wb') - ini_f.puts(posix_ini.to_s) + ini_f.puts(r.posix_ini.to_s) ini_f.close() ini_f = File.open(File.join(telemetry_folder, 'Windows.ini'), 'wb') - ini_f.puts(posix_ini.to_s) + ini_f.puts(r.windows_ini.to_s) ini_f.close() - r = PhptTestResult::Array.new() # TODO - # write list of extensions tested - write_ext_file(r.ext(:run), 'EXT_RUN.list') - write_ext_file(r.ext(:all), 'EXT_ALL.list') - write_ext_file(r.ext(:skip), 'EXT_SKIP.list') - - # write list of scenarios tested - f = File.open(File.join(telemetry_folder, 'scenarios.list'), 'wb') - scenarios.values.each do |scn| - f.puts(scn.scn_name) - end - f.close() + write_ext_file(telemetry_folder, r.ext(:run), 'EXT_RUN.list') + write_ext_file(telemetry_folder, r.ext(:all), 'EXT_ALL.list') + write_ext_file(telemetry_folder, r.ext(:skip), 'EXT_SKIP.list') - # write system info too - f = File.open(File.join(telemetry_folder, 'systeminfo.txt'), 'wb') - f.puts(@host.systeminfo) - f.close() - - report = create_per_report() - - # TODO feed results and INIs to report - - test_ctx.semaphore4.synchronize do - report.text_print() - - # - # - if $interactive_mode - if test_ctx.first_run(@host, @php, @middleware) - if test_ctx.prompt_yesno('PFTT: Re-run and compare the results to first run?') - test_ctx.rerun - end - else - if test_ctx.prompt_yesno('PFTT: Re-run and compare the results to this run?') - test_ctx.set_current_as_first_run(@host, @php, @middleware, self) - test_ctx.rerun - end - end - end - # - # - - end + return report end - def do_single_test_case(deploydir, deployed, skip_if_code_cache, skip_if_result_cache, test_ctx, test_cases, test_case, tmpdir, fs_scn, scenarios) + def do_single_test_case(php, host, middleware, deploydir, deployed, skip_if_code_cache, skip_if_result_cache, test_ctx, test_cases, test_case, tmpdir, fs_scn, scenarios) tmiddleware = @middleware.clone # important: some PHPT tests use paths relative to their deployment location (must change CWD) @@ -390,13 +343,12 @@ module TestBench deployed[:file] = deploy_phpt_section test_case, :file out_err = '' - #begin + begin out_err = tmiddleware.execute_php_script( deployed[:file], test_case, :test, scenarios )[1] - #rescue - #puts $!.to_s - #out_err = 'PFTT-Internal-Error(execute_phpt_script:test): '+$!.to_s - # TODO end + rescue + test_ctx.add_exception(host, php, middleware, $!) + end throw :result, [PhptTestResult::Meaningful, out_err] ensure @@ -421,14 +373,14 @@ module TestBench # run of this test case done # # display, report and store result of this test case - # TODO begin + begin result = nil test_ctx.semaphore2.synchronize{ # don't modify result_spec, its cached/shared with other threads a = result_spec[0] # take the caught result and build the proper object out of it - result = a.new( test_case, self, deploydir, *result_spec[1...result_spec.length] ) + result = a.new( test_case, self, deploydir, @php, *result_spec[1...result_spec.length] ) } if result # generate the diff here in the thread unlocked @@ -437,29 +389,23 @@ module TestBench end + # lookup Legend Label for this host/php/middleware combination + label = test_ctx.legend_label(@host, @php, @middleware) - # TODO include host, middleware, php, scenarios - # assign legend number to each and just include numbers here - - test_ctx.console_out(" [#{result.status.to_s.upcase}] #{@self} #{test_case.relative_path}") + test_ctx.console_out(" [#{label}] [#{result.status.to_s.upcase}] #{@self} #{test_case.relative_path}") test_ctx.semaphore5.synchronize{ # save result to database # TODO result.insert(global_db, local_db, global_iter_id, local_iter_id) - - # write to flat file too - result.save } # TODO can't have @host or @php or @middleware test_ctx.add_result(@host, @php, @middleware, scenarios, result) end - #rescue - #end + rescue + end # -# rescue -# puts 'PFTT: phpt-error: '+$!.to_s - end # end def do_single_test_case + end # def do_single_test_case end # end class Phpt diff --git a/PFTT/lib/test_bench/wcat.rb b/PFTT/lib/test_bench/wcat.rb index a158393..c98a25e 100644 --- a/PFTT/lib/test_bench/wcat.rb +++ b/PFTT/lib/test_bench/wcat.rb @@ -1,6 +1,8 @@ module TestBench class Wcat < Base + requirement :platform => :windows + def run(local_host, hosts, perf_case, target_host) # test with 8, 16, and 32 virtual clients for each physical host do_run(local_host, hosts, perf_case.clone, target_host, 8) @@ -21,7 +23,9 @@ module TestBench check_run_content = true if not perf_case.first_two_comparisons(original_content, original_content2) - # TODO results.first_two_loads_mismatched = true # means we can't tell if page load failed other than by return code + # means we can't tell if page load failed other than by return code + results.first_two_loads_mismatched = true + check_run_content = false end @@ -36,11 +40,13 @@ module TestBench # execute wcctl (WCAT) which will wait for wcclient from hosts to connect localhost.exec("#{wcat_path.convert_path}\wcctl.exe -t #{client_path} -f #{settings_path} -s #{target_host.host}:#{target_host.port} -v #{clients_per_host} -c 1 -o #{log_path} -x") + wcat_controller_machine_name = localhost.hostname + # execute wcclient on each host which will connect to wcctl on this host and then begin # running the performance test hosts.each{|host| # exec wcclient in another thread - host.exec("wcclient.exe #{wcat_controller_machine_name}") # TODO + host.exec("wcclient.exe #{wcat_controller_machine_name}") } # @@ -68,8 +74,9 @@ module TestBench end def get_web_page(perf_case) - # TODO - [200, ''] + http_response = Net::HTTP.get_response(URI.parse(perf_case.url_path)) + + [http_response.code, http_response.body ] end def settings(perf_case) diff --git a/PFTT/lib/test_result/perf_result.rb b/PFTT/lib/test_result/perf_result.rb index acde230..cb0df8a 100644 --- a/PFTT/lib/test_result/perf_result.rb +++ b/PFTT/lib/test_result/perf_result.rb @@ -4,11 +4,13 @@ module TestResult class MultiAppPerfResult class AppPerfResult class OSPerfResult + # means we can't tell if page load failed other than by return code + attr_accessor :first_two_loads_mismatched attr_reader :runs def initialize @runs = [] - + @first_two_loads_mismatched end class PerfRunResult diff --git a/PFTT/lib/util.rb b/PFTT/lib/util.rb index aa2eef7..cedd29f 100644 --- a/PFTT/lib/util.rb +++ b/PFTT/lib/util.rb @@ -3,6 +3,23 @@ module Util end +def os_short_name(os) + os.sub!('Windows', 'Win') + os.sub!('RTM', 'SP0') + + # remove common words + os.sub!('Professional', '') + os.sub!('Standard', '') + os.sub!('Enterprise', '') + os.sub!('Microsoft', '') + os.sub!('PC', '') + os.sub!('-based', '') + os.sub!(' ', ' ') + os.sub!(' ', ' ') + + return os.lstrip.rstrip +end + def cd_php_sdk(host) if host.posix? unless host.exist?('$HOME/php-sdk') diff --git a/PFTT/lib/util/column_manager.rb b/PFTT/lib/util/column_manager.rb index 0c48693..6a7ae67 100644 --- a/PFTT/lib/util/column_manager.rb +++ b/PFTT/lib/util/column_manager.rb @@ -24,6 +24,15 @@ module Util # :bgcolor #000000 - hex color to use as the background color of the cell # end + + protected + + attr_accessor :row_num + + def clone(cm) + cm.row_num = row_num + end + end end end diff --git a/PFTT/lib/util/column_manager/html.rb b/PFTT/lib/util/column_manager/html.rb index 66ff1f6..7de17bb 100644 --- a/PFTT/lib/util/column_manager/html.rb +++ b/PFTT/lib/util/column_manager/html.rb @@ -9,7 +9,7 @@ module Util end def clone - Util::ColumnManager::Html.new + clone(Util::ColumnManager::Html.new()) end def render diff --git a/PFTT/lib/util/column_manager/text.rb b/PFTT/lib/util/column_manager/text.rb index 1b46c74..5af0154 100644 --- a/PFTT/lib/util/column_manager/text.rb +++ b/PFTT/lib/util/column_manager/text.rb @@ -15,7 +15,7 @@ module Util end def clone - Util::ColumnManager::Text.new(@column_count) + clone(Util::ColumnManager::Text.new(@column_count)) end def add_row(*in_values) diff --git a/PFTT/lib/util/install.rb b/PFTT/lib/util/install.rb index 1292ad3..0b05eee 100644 --- a/PFTT/lib/util/install.rb +++ b/PFTT/lib/util/install.rb @@ -65,23 +65,30 @@ def linux_install(host, cmd, linux_name) end end -def win_install(host, cmd, win_installer_server_path, win_installer_options) +def win_install_installed?(cmd) out_err, status = host.exec!("WHERE #{cmd}") - - if status == 0 - # already installed - return - end - + + return status == 0 +end + +def win_install_copy(host, win_installer_server_path) # download installer into __DIR__/deps folder host.exec!("NET USE R: \\#{$dep_server_share}\ /user:#{$dep_server_user} /persistent:no #{$dep_server_password}") - + local_win_install_path = "#{__DIR__}/deps/#{win_installer_server_path}" - + host.exec!("COPY R:/PFTT/DEPS/#{win_installer_server_path} #{local_win_install_path}") - - host.exec!("NET USE R: /D") + host.exec!("NET USE R: /D") +end + +def win_install(host, cmd, win_installer_server_path, win_installer_options) + if win_install_installed?(cmd) + return + end + + win_install_copy(win_installer_server_path) + # run installer host.exec!("#{local_win_install_path} #{win_installer_options}") end diff --git a/PFTT/scripts/SDK4Win/PHP SDK Shell.lnk b/PFTT/scripts/SDK4Win/PHP SDK Shell.lnk index b9ddb2c..5f5514d 100644 Binary files a/PFTT/scripts/SDK4Win/PHP SDK Shell.lnk and b/PFTT/scripts/SDK4Win/PHP SDK Shell.lnk differ diff --git a/PFTT/scripts/SDK4Win/SetupPHPSDK.cmd b/PFTT/scripts/SDK4Win/SetupPHPSDK.cmd index cbc1454..03eb8e9 100644 --- a/PFTT/scripts/SDK4Win/SetupPHPSDK.cmd +++ b/PFTT/scripts/SDK4Win/SetupPHPSDK.cmd @@ -6,9 +6,9 @@ set PHP_SDK=%SYSTEMDRIVE%\php-sdk set PHPT_BRANCH=%PHP_SDK%\svn\branches set PHP_BIN=%PHP_SDK%\bin set PHP_BUILDS=%PHP_SDK%\builds -set PFTT_HOME=%PHP_SDK%\PFTT +set PFTT_HOME=%PHP_SDK%\PFTT2\PFTT REM TODO TEMP use regular PFTT_HOME -set PFTT_HOME=C:\Users\v-mafick\Desktop\sf\workspace\PFTT +set PFTT_HOME=%PHP_SDK%\0\PFTT2\PFTT set PFTT_RESULTS=%PHP_SDK%\PFTT-Results set PFTT_SCRIPTS=%PHP_SDK%\PFTT-Scripts set PFTT_PHPS=%PHP_SDK%\PFTT-PHPs @@ -32,7 +32,7 @@ REM "%ProgramFiles%\Microsoft SDKs\Windows\v7.0\Bin\SetEnv.cmd" /xp /x86 /releas REM %PHP_BIN%\phpsdk_setvars.bat -cd %PHP_SDK% +cd %PFTT_HOME% welcome