~ubuntu-branches/ubuntu/precise/slurm-llnl/precise

#!/usr/bin/expect
############################################################################
# Purpose: Test of SLURM functionality
#          Test of sinfo cpu total and allocated
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2009 Lawrence Livermore National Security.
# Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
# Written by Joseph Donaghy <donaghy1@llnl.gov>
# CODE-OCEC-09-009. All rights reserved.
#
# This file is part of SLURM, a resource management program.
# For details, see <https://computing.llnl.gov/linux/slurm/>.
# Please also read the included file: DISCLAIMER.
#
# SLURM is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with SLURM; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals

set test_id     "4.12"
set file_in     "test$test_id.in"
set exit_code   0
set pnumsc      0
set pnumsi      0
set aprocsc     0
set aprocsi     0
set inode_found 0
set inode_name  ""
set smallest    1
set layout "static"
set select_type ""
set node_scaling 0

proc scontrol_test { node proc_cnt } {
	global scontrol number prompt
	upvar spawn_id spawn_id

	set found 0
	set rc 0

	send "$scontrol show node $node\r"
	expect {
		-re "CPUAlloc=($number)" {
			set num_alloc $expect_out(1,string)
			set found 1
			if {$proc_cnt != $num_alloc} {
				send_user "\nFAILURE: requested $proc_cnt but got $num_alloc instead\n"
				set rc 1
			}
			exp_continue
		}
		-re $prompt {
		}
		timeout {
			send_user "\nFAILURE: scontrol not responding\n"
			slow_kill $mypid
			return 1
		}
		eof {
			wait
		}
	}

	if {!$found} {
		send_user "\nFAILURE: didn't get expected output from scontrol\n"
		set rc 1
	}

	return $rc
}

proc sinfo_test_1 { node proc_cnt total_procs idle_cpus } {
	global sinfo number prompt
	upvar spawn_id spawn_id

	set found 0
	set rc 0

	send "$sinfo -o \"%C %A %N\" -h -n $node\r"
	# make sure we get by the sinfo command so we don't
	# think the %'s are a prompt
	expect {
		"%C %A %N" {
		}
	}
	expect {
		-re "($number)(K?).($number)(K?).($number)(K?).($number)(K?) ($number)(K?).($number)(K?) $node" {
			set found 1
			set num_alloc $expect_out(1,string)
			if {[string compare $expect_out(2,string) ""]} {
				set num_alloc [expr $num_alloc * 1024]
			}
			set num_idle $expect_out(3,string)
			if {[string compare $expect_out(4,string) ""]} {
				set num_idle [expr $num_idle * 1024]
			}
			set num_other $expect_out(5,string)
			if {[string compare $expect_out(6,string) ""]} {
				set num_other [expr $num_other * 1024]
			}
			set num_total $expect_out(7,string)
			if {[string compare $expect_out(8,string) ""]} {
				set num_total [expr $num_total * 1024]
			}

			if { $num_alloc != $proc_cnt } {
				send_user "\nFAILURE: sinfo 1 allocated cpus wrong, got $num_alloc but needed $proc_cnt\n"
				set rc 1
			} elseif { $num_idle != $idle_cpus } {
				send_user "\nFAILURE: sinfo 1 idle cpus wrong, got $num_idle but needed $idle_cpus\n"
				set rc 1
			} elseif { $num_total != $total_procs } {
				send_user "\nFAILURE: sinfo 1 total cpus wrong, got $num_total but needed $total_procs\n"
				set rc 1
			}
			exp_continue
		}
		-re $prompt {
		}
		timeout {
			send_user "\nFAILURE: sinfo not responding\n"
			slow_kill $mypid
			return 1
		}
		eof {
			wait
		}
	}

	if {!$found} {
		send_user "\nFAILURE: didn't get expected output from sinfo\n"
		set rc 1
	}

	return $rc
}

proc sinfo_test_2 { node proc_cnt total_procs } {
	global sinfo number prompt node_scaling
	upvar spawn_id spawn_id

	set rc 0
	set num_alloc 0
	set num_idle 0
	set alloc_nodes 1
	set total_nodes 1

	if {$node_scaling} {
		set alloc_nodes [expr $proc_cnt / $node_scaling]
		set total_nodes [expr $total_procs / $node_scaling]
	}
	set idle_nodes [expr $total_nodes - $alloc_nodes]

	send "$sinfo -o \"%t %D %N\" -h -n $node\r"
	# make sure we get by the sinfo command so we don't
	# think the %'s are a prompt
	expect {
		"%t %D %N" {
		}
	}
	expect {
		-re "alloc ($number)(K?) $node" {
			set num_alloc $expect_out(1,string)
			if {[string compare $expect_out(2,string) ""]} {
				set num_alloc [expr $inode_procs * 1024]
			}
			exp_continue
		}
		-re "idle ($number)(K?) $node" {
			set num_idle $expect_out(1,string)
			if {[string compare $expect_out(2,string) ""]} {
				set num_idle [expr $num_idle * 1024]
			}
			exp_continue
		}
		-re $prompt {
		}
		timeout {
			send_user "\nFAILURE: sinfo not responding\n"
			slow_kill $mypid
			return 1
		}
		eof {
			wait
		}
	}

	if { $num_alloc != $alloc_nodes } {
		send_user "\nFAILURE: sinfo 2 allocated nodes wrong, got $num_alloc but needed $alloc_nodes\n"
		set rc 1
	} elseif { $num_idle != $idle_nodes } {
		send_user "\nFAILURE: sinfo 2 idle nodes wrong, got $num_idle but needed $idle_nodes\n"
		set rc 1
	}

	return $rc
}

# allocate a set of nodes (node_cnt) and the quit right after
proc allocate_and_quit { node proc_cnt total_procs } {
	global salloc scontrol sinfo number alpha_numeric_under
	global prompt select_type procs_per_node

	set job_id 0
	set num_alloc 0
	set block ""
	set rc 0
	set timeout 60
	set idle_cpus [expr $total_procs - $proc_cnt]

	set mypid [spawn $salloc -w $node -N1 -n $proc_cnt bash]
	expect {
		-re "Granted job allocation ($number)" {
			set job_id $expect_out(1,string)
			exp_continue
		}

		-re $prompt {
			# test for scontrol to give me the correct cpu count
			if { [scontrol_test $node $proc_cnt] } {
				send "exit\r"
				return 1
			}


			# test for sinfo to give me the correct cpu count
			if { [sinfo_test_1 $node $proc_cnt $total_procs $idle_cpus] } {
				send "exit\r"
				return 1
			}
			# test for sinfo to give me the correct node count
			if { [sinfo_test_2 $node $proc_cnt $total_procs] } {
				send "exit\r"
				return 1
			}
			send "exit\r"
			exp_continue
		}

		-re "Unable to contact" {
			send_user "\nFAILURE: slurm appears to be down\n"
			exp_continue
		}
		timeout {
			send_user "\nFAILURE: salloc not responding\n"
			if {$job_id != 0} {
				cancel_job $job_id
			}
			slow_kill $mypid
			return 1
		}
		eof {
			wait
		}
	}

	return $rc
}

############################################################################
# test starts here
############################################################################

print_header $test_id

# find the default partition
set def_part [default_partition]

# find the nodes in the default partition
log_user 0
set def_hostlist ""
set part_exclusive 0
spawn $scontrol show part $def_part
expect {
	-re " Shared=EXCLUSIVE" {
		set part_exclusive 1
		exp_continue
	}
	-re " Nodes=($alpha_numeric_nodelist)"  {
		set def_hostlist $expect_out(1,string)
		exp_continue
	}
	-re " BasePartitions=($alpha_numeric_nodelist)" {
		set def_hostlist $expect_out(1,string)
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: scontrol not responding\n"
		set exit_code 1
	}
	eof {
		wait
	}
}
set host_cnt 0
spawn $scontrol show hostnames $def_hostlist
expect {
	-re "($alpha_numeric_under)"  {
		set host_name($host_cnt) $expect_out(1,string)
		incr host_cnt
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: scontrol not responding\n"
		set exit_code 1
	}
	eof {
		wait
	}
}
log_user 1
if {$host_cnt == 0} {
	send_user "\nFAILURE: could not find any nodes in default partition\n"
	exit 1
}

# find me an idle node in default partition
log_user 0
set inode_name ""
set inode_cores_per_socket 0
set inode_procs 0
set units ""
set inode_sockets 0
set inode_threads_per_core 0

set fd [open "|$scontrol --oneliner show node $def_hostlist"]
exp_internal 1
while {[gets $fd line] != -1} {
	if {[regexp {NodeName=(\w+).*CoresPerSocket=(\d+).*CPUTot=(\d+)(K?).*Sockets=(\d+) State=IDLE ThreadsPerCore=(\d+)} $line frag inode_name inode_cores_per_socket inode_procs units inode_sockets inode_threads_per_core] == 1} {
		break
	}
}
exp_internal 0
if {[string compare $units ""]} {
	set inode_procs [expr $inode_procs * 1024]
}
catch {close $fd}

log_user 1

if {!$inode_procs} {
	send_user "\nFAILURE: couldn't find an idle node in the default partition\n"
	exit 1
}

send_user "found idle node $inode_name with $inode_procs processors\n"

# figure out the select plugin we are using
set select_type [test_select_type]
if {![string compare $select_type "bluegene"]} {
	# figure out some things if a bluegene system
	set layout [get_bluegene_layout]
	if {$layout == 0} {
		send_user "\nFAILURE: No layout mode found for this system\n"
		exit 1
	}
	set psets [get_bluegene_psets]

	if {$psets == 0} {
		send_user "\nFAILURE: No psets are set on this system\n"
		exit 1
	}
	set type [get_bluegene_type]

	if {$type == 0} {
		send_user "\nFAILURE: No bluegene type found \n"
		exit 1
	}

	if {![string compare $type "P"]} {
		if {$psets >= 32} {
			set smallest 16
		} elseif {$psets >= 16} {
			set smallest 32
		} elseif {$psets >= 8} {
			set smallest 64
		} else {
			set smallest 128
		}
	} elseif {![string compare $type "L"]} {
		if {$psets >= 16} {
			set smallest 32
		} else {
			set smallest 128
		}
	} else {
		send_user "\nFAILURE: unknown bluegene system type '$type'\n";
		exit 1
	}
	set node_scaling [get_bluegene_procs_per_cnode]
	set smallest [expr $smallest * $node_scaling]
} elseif {![string compare $select_type "linear"]} {
	set smallest $inode_procs
} else {
	set select_params [test_select_type_params]
	if {$part_exclusive == 1} {
		set smallest $inode_procs
	} elseif {![string compare $select_params "CR_CPU"]} {
		set smallest $inode_threads_per_core
	} elseif {![string compare $select_params "CR_CPU_MEMORY"]} {
		set smallest $inode_threads_per_core
	} elseif {![string compare $select_params "NONE"]} {
		set smallest $inode_threads_per_core
	} elseif {![string compare $select_params "CR_CORE"]} {
		set smallest $inode_threads_per_core
	} elseif {![string compare $select_params "CR_CORE_MEMORY"]} {
		set smallest $inode_threads_per_core
	} elseif {![string compare $select_params "CR_SOCKET"]} {
		set smallest [expr $inode_cores_per_socket *$inode_threads_per_core]
	} elseif {![string compare $select_params "CR_SOCKET_MEMORY"]} {
		set smallest [expr $inode_cores_per_socket *$inode_threads_per_core]
	} else {
		send_user "\nWARNING: failed to parse SelectTypeParameters '$select_params'\n"
		set smallest $inode_procs
	}
}

set exit_code [allocate_and_quit $inode_name $smallest $inode_procs]
if {!$exit_code && $smallest != $inode_procs} {
	# just to make sure we get a clean state we will sleep a bit
	sleep 1
	set exit_code [allocate_and_quit $inode_name $inode_procs $inode_procs]
}
if {$exit_code == 0} {
	send_user "\nSUCCESS\n"
} else {
	exit $exit_code
}