1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417
|
/* Copyright (c) 2009 Peter Troshin
*
* JAva Bioinformatics Analysis Web Services (JABAWS) @version: 1.0
*
* This library is free software; you can redistribute it and/or modify it under the terms of the
* Apache License version 2 as published by the Apache Software Foundation
*
* This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
* even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the Apache
* License for more details.
*
* A copy of the license is in apache_license.txt. It is also available here:
* @see: http://www.apache.org/licenses/LICENSE-2.0.txt
*
* Any republication or derived work distributed in source code form
* must include this copyright and license notice.
*/
package compbio.engine.client;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.security.InvalidParameterException;
import java.util.Map;
import org.apache.log4j.Logger;
import compbio.engine.client.Executable.ExecProvider;
import compbio.engine.conf.DirectoryManager;
import compbio.engine.conf.PropertyHelperManager;
import compbio.metadata.JobStatus;
import compbio.metadata.LimitsManager;
import compbio.metadata.ResultNotAvailableException;
import compbio.util.FileUtil;
import compbio.util.PropertyHelper;
import compbio.util.SysPrefs;
public final class EngineUtil {
private static final PropertyHelper ph = PropertyHelperManager
.getPropertyHelper();
private static final Logger log = Logger.getLogger(EngineUtil.class);
public static boolean isValidJobId(final String key) {
if (compbio.util.Util.isEmpty(key)) {
return false;
}
int delIdx = key.indexOf(DirectoryManager.DELIM);
if (delIdx < 0) {
return false;
}
String id = key.substring(delIdx + DirectoryManager.DELIM.length());
try {
Long.parseLong(id);
} catch (NumberFormatException e) {
log.debug("Invalid key! " + e.getLocalizedMessage());
return false;
}
return true;
}
public static void writeStatFile(String workDirectory,
String fileAndEventName) {
// never override old stat files!
// Work directory could be null for cancelled or incomplete jobs, just
// ignore
if (!compbio.util.Util.isEmpty(workDirectory)) {
writeFile(workDirectory, fileAndEventName,
new Long(System.currentTimeMillis()).toString(), false);
}
}
public static void writeFile(String workDirectory, String fileAndEventName,
String content, boolean override) {
File file = null;
if (compbio.util.Util.isEmpty(workDirectory)) {
log.debug("Calling compbio.engine.Util.writeFile() with not work directory."
+ " Skipping writing statistics!");
return;
}
assert !compbio.util.Util.isEmpty(content) : "Content expected!";
FileWriter writer = null;
try {
file = new File(workDirectory, fileAndEventName);
// Do not override existing files unless asked to do so !
if (file.exists() && !override) {
return;
}
writer = new FileWriter(file);
writer.write(content);
writer.close();
log.debug("File " + fileAndEventName + " with content: " + content
+ " has been recorder successfully! ");
} catch (IOException e) {
log.error("Could not record the " + fileAndEventName + " file in "
+ workDirectory + " for local execution! Ignoring... "
+ e.getMessage());
} finally {
FileUtil.closeSilently(log, writer);
}
}
public static final boolean writeMarker(String workDirectory,
JobStatus fileType) {
if (fileType == null) {
throw new NullPointerException("MarkerType must be provided!");
}
if (fileType == fileType.FINISHED || fileType == fileType.STARTED) {
throw new IllegalArgumentException(
"Please use Util.writeStatFile(workDirectory, fileAndEventName) to record FINISHED and STARTED statuses!");
}
if (!PathValidator.isValidDirectory(workDirectory)) {
// This is OK as some task could be cancelled even before they
// started
log.warn("Attempting to write " + fileType
+ " marker in the work directory " + workDirectory
+ " is not provided or does not exist!");
return false;
}
try {
File sfile = new File(workDirectory, fileType.toString());
if (!sfile.exists()) {
return sfile.createNewFile();
}
} catch (IOException e) {
log.error(
"Could not record stat marker file " + fileType
+ " into the directory " + workDirectory + " ! "
+ e.getMessage(), e.getCause());
}
return false;
}
public static boolean isMarked(String workDirectory, JobStatus marker) {
if (!PathValidator.isValidDirectory(workDirectory)) {
throw new NullPointerException("Work directory " + workDirectory
+ " is not provided or does not exist!");
}
return new File(workDirectory, marker.toString()).exists();
}
public static Map<String, String> mergeEnvVariables(
final Map<String, String> sysEnvTobeModified,
final Map<String, String> variables) {
if (variables.containsKey(EnvVariableProcessor.PATH)) {
String propPath = variables.get(EnvVariableProcessor.PATH);
String sysPATH = sysEnvTobeModified.get(EnvVariableProcessor.PATH);
String syspath = sysEnvTobeModified.get(EnvVariableProcessor.PATH
.toLowerCase());
// This version appears surprisingly often on windows machines
boolean added = false;
String sysPath = sysEnvTobeModified.get("Path");
if (sysPATH != null) {
sysEnvTobeModified.put(EnvVariableProcessor.PATH, sysPATH
+ File.pathSeparator + propPath);
added = true;
}
if (syspath != null) {
sysEnvTobeModified.put(EnvVariableProcessor.PATH.toLowerCase(),
syspath + File.pathSeparator + propPath);
added = true;
}
if (sysPath != null) {
sysEnvTobeModified.put("Path", sysPath + File.pathSeparator
+ propPath);
added = true;
}
// If not path variable is found, then add it
if (!added) {
sysEnvTobeModified.put(EnvVariableProcessor.PATH, propPath);
}
variables.remove(EnvVariableProcessor.PATH);
}
sysEnvTobeModified.putAll(variables);
return sysEnvTobeModified;
}
public static String convertToAbsolute(String relativePath) {
// If specified path is relative, than make it absolute
String absolute = relativePath;
if (!PathValidator.isAbsolutePath(relativePath)) {
absolute = PropertyHelperManager.getLocalPath() + relativePath;
EngineUtil.log.trace("Changing local path in enviromental variable to absolute: FROM "
+ relativePath + " TO " + absolute);
}
return absolute;
}
public static String getExecProperty(String propertySpec, Executable<?> exec) {
assert !compbio.util.Util.isEmpty(propertySpec);
assert exec != null;
return EngineUtil.getExecProperty(propertySpec, exec.getClass());
}
public static String getExecProperty(String propertySpec, Class<?> clazz) {
assert !compbio.util.Util.isEmpty(propertySpec);
assert clazz != null;
String property = clazz.getSimpleName().toLowerCase() + "."
+ propertySpec.toLowerCase();
log.trace("Processing property: " + property);
return ph.getProperty(property);
}
public static String getFullPath(String workDirectory, String fileName) {
assert !compbio.util.Util.isEmpty(fileName) : "Filename must be provided! ";
assert !compbio.util.Util.isEmpty(workDirectory) : "Workdirectory must be provided! ";
return workDirectory + File.separator + fileName;
}
public static String getCommand(ExecProvider provider, Class<?> clazz) {
if (provider == ExecProvider.Any) {
throw new IllegalArgumentException(
"A particular execution environment must be chosen");
}
String execCommandName = clazz.getSimpleName().toLowerCase();
String bin = "";
if (provider == ExecProvider.Local) {
if (SysPrefs.isWindows) {
bin = ph.getProperty("local." + execCommandName
+ ".bin.windows");
} else {
bin = ph.getProperty("local." + execCommandName + ".bin");
}
// For executable Jar files the location of Java executable is not
// required for local execution. If it is not provided, JABAWS will
// attempt to use Java from JAVA_HOME env variable
if (isJavaLibrary(clazz)) {
if (compbio.util.Util.isEmpty(bin)) {
bin = getJava();
}
}
// If path to executable defined in the properties is not absolute,
// then make it so
// as setting working directory of ProcessBuilder will make it
// impossible
// to find an executable otherwise
if (!compbio.util.Util.isEmpty(bin)
&& !PathValidator.isAbsolutePath(bin)) {
bin = bin.trim();
if (bin.equalsIgnoreCase("java")
|| bin.equalsIgnoreCase("java.exe")) {
// do not make path absolute to the java executable if
// relative path is provided. Java executable is not a part
// of JABAWS distribution!
} else {
bin = PropertyHelperManager.getLocalPath() + bin;
}
}
} else {
bin = ph.getProperty("cluster." + execCommandName + ".bin");
}
// Could have done: Set executable flag if not set
// but - do not because in some cases more than one file must be made
// executable!
/*
* if (!compbio.util.Util.isEmpty(bin)) { File command = new File(bin);
* if (!command.canExecute()) { log.debug(
* "The command line binary is not executable! (just unpacked from war file? )"
* ); log.debug("Attempting to set executable flag for command: " +
* bin); command.setExecutable(true, false); } }
*/
log.debug("Using executable: " + bin);
return bin; // File.separator
}
/**
* Returns true of executableName.jar.file property has some value in the
* Executable.properties file, false otherwise.
*
* @param clazz
* @return
*/
public static boolean isJavaLibrary(Class<?> clazz) {
String execCommandName = clazz.getSimpleName().toLowerCase();
String java_lib = ph.getProperty(execCommandName + ".jar.file");
return !compbio.util.Util.isEmpty(java_lib);
}
/**
* Returns the absolute path to the Java executable from JAVA_HOME
*
* @return returns the absolute path to the Java executable from JAVA_HOME
*/
public static String getJava() {
String javahome = System.getProperty("java.home");
if (compbio.util.Util.isEmpty(javahome)) {
javahome = System.getenv("JAVA_HOME");
}
if (compbio.util.Util.isEmpty(javahome)) {
log.warn("Cannot find Java in java.home system property "
+ "or JAVA_HOME environment variable! ");
return null;
}
File jh = new File(javahome);
if (jh.exists() && jh.isDirectory()) {
String java = javahome + File.separator + "bin" + File.separator
+ "java";
if (SysPrefs.isWindows) {
java += ".exe";
}
File jexe = new File(java);
if (jexe.exists() && jexe.isFile() && jexe.canExecute()) {
log.info("Using Java from: " + jexe.getAbsolutePath());
return jexe.getAbsolutePath();
} else {
log.warn("Cannot find java executable in the JAVA_HOME!");
}
} else {
log.warn("JAVA_HOME does not seems to point to a valid directory! Value: "
+ javahome);
}
return null;
}
public static ExecProvider getSupportedRuntimes(Class<?> clazz) {
boolean localRuntimeSupport = false;
boolean clusterRuntimeSupport = false;
String executableName = clazz.getSimpleName().toLowerCase();
String localRuntime1 = ph.getProperty("local." + executableName
+ ".bin.windows");
String localRuntime2 = ph.getProperty("local." + executableName
+ ".bin");
if (!compbio.util.Util.isEmpty(localRuntime1)
|| !compbio.util.Util.isEmpty(localRuntime2)) {
localRuntimeSupport = true;
} else {
localRuntimeSupport = isJavaLibrary(clazz) && getJava() != null;
}
String clusterRuntime = ph.getProperty("cluster." + executableName
+ ".bin");
if (!compbio.util.Util.isEmpty(clusterRuntime)) {
clusterRuntimeSupport = true;
}
if (localRuntimeSupport && clusterRuntimeSupport) {
return ExecProvider.Any;
} else if (localRuntimeSupport) {
return ExecProvider.Local;
} else if (clusterRuntimeSupport) {
return ExecProvider.Cluster;
}
// Means executable cannot be executed -> is improperly configured
// should be ignored
throw new InvalidParameterException(
"Executable is not provided for any runtime environments");
}
public static ConfiguredExecutable<?> loadExecutable(String taskId)
throws ResultNotAvailableException {
String workDir = compbio.engine.Configurator.getWorkDirectory(taskId);
// The results for this job has been collected once, or the JVM may
// have been restarted,
// so that the job is not in the job list
// ->load a ConfiguredExercutable from saved run and return it
FileInputStream fileInStream = null;
ConfiguredExecutable<?> exec = null;
try {
fileInStream = new FileInputStream(workDir + File.separator
+ RunConfiguration.rconfigFile);
RunConfiguration rconf = RunConfiguration.load(fileInStream);
exec = ConfExecutable.newConfExecutable(rconf);
fileInStream.close();
} catch (FileNotFoundException e) {
log.error(
"Could not find run configuration to load!"
+ e.getLocalizedMessage(), e.getCause());
throw new ResultNotAvailableException(
"Could not find run configuration to load!"
+ e.getMessage(), e.getCause());
} catch (IOException e) {
log.error(
"IO Exception while reading run configuration file!"
+ e.getLocalizedMessage(), e.getCause());
throw new ResultNotAvailableException(
"Could not load run configuration!" + e.getMessage(),
e.getCause());
} finally {
FileUtil.closeSilently(log, fileInStream);
}
return exec;
}
/**
* For now just assume that all parameters which came in needs setting it
* will be a client responsibility to prepare RunnerConfig object then
*
* @param rconfig
* @return
*
* public static List<String> toOptionString(RunnerConfig<?>
* rconfig) { String option = ""; List<String> options = new
* ArrayList<String>(); for (Parameter<?> par :
* rconfig.getParameters()) { if (par.getPossibleValues().isEmpty())
* { option = par.getOptionName(); } else { option =
* par.getOptionName() + "=" + par.getPossibleValues().get(0); } //
* separate options options.add(option); } return options; }
*/
public static <T> LimitsManager<T> getLimits(Class<T> clazz) {
LimitsManager<T> limits = null;
try {
limits = ConfExecutable.getRunnerLimits(clazz);
} catch (FileNotFoundException e) {
EngineUtil.log.warn("No limits are found for " + clazz + " executable! "
+ e.getLocalizedMessage(), e.getCause());
// its ok, limit may not be initialized
} catch (IOException e) {
EngineUtil.log.warn("IO exception while attempting to read limits for "
+ clazz + " executable! " + e.getLocalizedMessage(),
e.getCause());
}
return limits;
}
}
|