001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.hadoop.lib.service.hadoop; 020 021import org.apache.hadoop.classification.InterfaceAudience; 022import org.apache.hadoop.conf.Configuration; 023import org.apache.hadoop.fs.CommonConfigurationKeysPublic; 024import org.apache.hadoop.fs.FileSystem; 025import org.apache.hadoop.fs.Path; 026import org.apache.hadoop.fs.permission.FsPermission; 027import org.apache.hadoop.lib.server.BaseService; 028import org.apache.hadoop.lib.server.ServiceException; 029import org.apache.hadoop.lib.service.FileSystemAccess; 030import org.apache.hadoop.lib.service.FileSystemAccessException; 031import org.apache.hadoop.lib.service.Instrumentation; 032import org.apache.hadoop.lib.service.Scheduler; 033import org.apache.hadoop.lib.util.Check; 034import org.apache.hadoop.lib.util.ConfigurationUtils; 035import org.apache.hadoop.security.UserGroupInformation; 036import org.apache.hadoop.util.StringUtils; 037import org.apache.hadoop.util.VersionInfo; 038import org.slf4j.Logger; 039import org.slf4j.LoggerFactory; 040 041import java.io.File; 042import java.io.IOException; 043import java.net.URI; 044import java.security.PrivilegedExceptionAction; 045import java.util.Collection; 046import java.util.HashSet; 047import java.util.Map; 048import java.util.Set; 049import java.util.concurrent.ConcurrentHashMap; 050import java.util.concurrent.TimeUnit; 051import java.util.concurrent.atomic.AtomicInteger; 052 053@InterfaceAudience.Private 054public class FileSystemAccessService extends BaseService implements FileSystemAccess { 055 private static final Logger LOG = LoggerFactory.getLogger(FileSystemAccessService.class); 056 057 public static final String PREFIX = "hadoop"; 058 059 private static final String INSTRUMENTATION_GROUP = "hadoop"; 060 061 public static final String AUTHENTICATION_TYPE = "authentication.type"; 062 public static final String KERBEROS_KEYTAB = "authentication.kerberos.keytab"; 063 public static final String KERBEROS_PRINCIPAL = "authentication.kerberos.principal"; 064 public static final String FS_CACHE_PURGE_FREQUENCY = "filesystem.cache.purge.frequency"; 065 public static final String FS_CACHE_PURGE_TIMEOUT = "filesystem.cache.purge.timeout"; 066 067 public static final String NAME_NODE_WHITELIST = "name.node.whitelist"; 068 069 public static final String HADOOP_CONF_DIR = "config.dir"; 070 071 private static final String[] HADOOP_CONF_FILES = {"core-site.xml", "hdfs-site.xml"}; 072 073 private static final String FILE_SYSTEM_SERVICE_CREATED = "FileSystemAccessService.created"; 074 075 private static class CachedFileSystem { 076 private FileSystem fs; 077 private long lastUse; 078 private long timeout; 079 private int count; 080 081 public CachedFileSystem(long timeout) { 082 this.timeout = timeout; 083 lastUse = -1; 084 count = 0; 085 } 086 087 synchronized FileSystem getFileSytem(Configuration conf) 088 throws IOException { 089 if (fs == null) { 090 fs = FileSystem.get(conf); 091 } 092 lastUse = -1; 093 count++; 094 return fs; 095 } 096 097 synchronized void release() throws IOException { 098 count--; 099 if (count == 0) { 100 if (timeout == 0) { 101 fs.close(); 102 fs = null; 103 lastUse = -1; 104 } 105 else { 106 lastUse = System.currentTimeMillis(); 107 } 108 } 109 } 110 111 // to avoid race conditions in the map cache adding removing entries 112 // an entry in the cache remains forever, it just closes/opens filesystems 113 // based on their utilization. Worse case scenario, the penalty we'll 114 // pay is that the amount of entries in the cache will be the total 115 // number of users in HDFS (which seems a resonable overhead). 116 synchronized boolean purgeIfIdle() throws IOException { 117 boolean ret = false; 118 if (count == 0 && lastUse != -1 && 119 (System.currentTimeMillis() - lastUse) > timeout) { 120 fs.close(); 121 fs = null; 122 lastUse = -1; 123 ret = true; 124 } 125 return ret; 126 } 127 128 } 129 130 public FileSystemAccessService() { 131 super(PREFIX); 132 } 133 134 private Collection<String> nameNodeWhitelist; 135 136 Configuration serviceHadoopConf; 137 138 private AtomicInteger unmanagedFileSystems = new AtomicInteger(); 139 140 private ConcurrentHashMap<String, CachedFileSystem> fsCache = 141 new ConcurrentHashMap<String, CachedFileSystem>(); 142 143 private long purgeTimeout; 144 145 @Override 146 protected void init() throws ServiceException { 147 LOG.info("Using FileSystemAccess JARs version [{}]", VersionInfo.getVersion()); 148 String security = getServiceConfig().get(AUTHENTICATION_TYPE, "simple").trim(); 149 if (security.equals("kerberos")) { 150 String defaultName = getServer().getName(); 151 String keytab = System.getProperty("user.home") + "/" + defaultName + ".keytab"; 152 keytab = getServiceConfig().get(KERBEROS_KEYTAB, keytab).trim(); 153 if (keytab.length() == 0) { 154 throw new ServiceException(FileSystemAccessException.ERROR.H01, KERBEROS_KEYTAB); 155 } 156 String principal = defaultName + "/localhost@LOCALHOST"; 157 principal = getServiceConfig().get(KERBEROS_PRINCIPAL, principal).trim(); 158 if (principal.length() == 0) { 159 throw new ServiceException(FileSystemAccessException.ERROR.H01, KERBEROS_PRINCIPAL); 160 } 161 Configuration conf = new Configuration(); 162 conf.set("hadoop.security.authentication", "kerberos"); 163 UserGroupInformation.setConfiguration(conf); 164 try { 165 UserGroupInformation.loginUserFromKeytab(principal, keytab); 166 } catch (IOException ex) { 167 throw new ServiceException(FileSystemAccessException.ERROR.H02, ex.getMessage(), ex); 168 } 169 LOG.info("Using FileSystemAccess Kerberos authentication, principal [{}] keytab [{}]", principal, keytab); 170 } else if (security.equals("simple")) { 171 Configuration conf = new Configuration(); 172 conf.set("hadoop.security.authentication", "simple"); 173 UserGroupInformation.setConfiguration(conf); 174 LOG.info("Using FileSystemAccess simple/pseudo authentication, principal [{}]", System.getProperty("user.name")); 175 } else { 176 throw new ServiceException(FileSystemAccessException.ERROR.H09, security); 177 } 178 179 String hadoopConfDirProp = getServiceConfig().get(HADOOP_CONF_DIR, getServer().getConfigDir()); 180 File hadoopConfDir = new File(hadoopConfDirProp).getAbsoluteFile(); 181 if (!hadoopConfDir.exists()) { 182 hadoopConfDir = new File(getServer().getConfigDir()).getAbsoluteFile(); 183 } 184 if (!hadoopConfDir.exists()) { 185 throw new ServiceException(FileSystemAccessException.ERROR.H10, hadoopConfDir); 186 } 187 try { 188 serviceHadoopConf = loadHadoopConf(hadoopConfDir); 189 } catch (IOException ex) { 190 throw new ServiceException(FileSystemAccessException.ERROR.H11, ex.toString(), ex); 191 } 192 193 LOG.debug("FileSystemAccess FileSystem configuration:"); 194 for (Map.Entry entry : serviceHadoopConf) { 195 LOG.debug(" {} = {}", entry.getKey(), entry.getValue()); 196 } 197 setRequiredServiceHadoopConf(serviceHadoopConf); 198 199 nameNodeWhitelist = toLowerCase(getServiceConfig().getTrimmedStringCollection(NAME_NODE_WHITELIST)); 200 } 201 202 private Configuration loadHadoopConf(File dir) throws IOException { 203 Configuration hadoopConf = new Configuration(false); 204 for (String file : HADOOP_CONF_FILES) { 205 File f = new File(dir, file); 206 if (f.exists()) { 207 hadoopConf.addResource(new Path(f.getAbsolutePath())); 208 } 209 } 210 return hadoopConf; 211 } 212 213 @Override 214 public void postInit() throws ServiceException { 215 super.postInit(); 216 Instrumentation instrumentation = getServer().get(Instrumentation.class); 217 instrumentation.addVariable(INSTRUMENTATION_GROUP, "unmanaged.fs", new Instrumentation.Variable<Integer>() { 218 @Override 219 public Integer getValue() { 220 return unmanagedFileSystems.get(); 221 } 222 }); 223 instrumentation.addSampler(INSTRUMENTATION_GROUP, "unmanaged.fs", 60, new Instrumentation.Variable<Long>() { 224 @Override 225 public Long getValue() { 226 return (long) unmanagedFileSystems.get(); 227 } 228 }); 229 Scheduler scheduler = getServer().get(Scheduler.class); 230 int purgeInterval = getServiceConfig().getInt(FS_CACHE_PURGE_FREQUENCY, 60); 231 purgeTimeout = getServiceConfig().getLong(FS_CACHE_PURGE_TIMEOUT, 60); 232 purgeTimeout = (purgeTimeout > 0) ? purgeTimeout : 0; 233 if (purgeTimeout > 0) { 234 scheduler.schedule(new FileSystemCachePurger(), 235 purgeInterval, purgeInterval, TimeUnit.SECONDS); 236 } 237 } 238 239 private class FileSystemCachePurger implements Runnable { 240 241 @Override 242 public void run() { 243 int count = 0; 244 for (CachedFileSystem cacheFs : fsCache.values()) { 245 try { 246 count += cacheFs.purgeIfIdle() ? 1 : 0; 247 } catch (Throwable ex) { 248 LOG.warn("Error while purging filesystem, " + ex.toString(), ex); 249 } 250 } 251 LOG.debug("Purged [{}} filesystem instances", count); 252 } 253 } 254 255 private Set<String> toLowerCase(Collection<String> collection) { 256 Set<String> set = new HashSet<String>(); 257 for (String value : collection) { 258 set.add(StringUtils.toLowerCase(value)); 259 } 260 return set; 261 } 262 263 @Override 264 public Class getInterface() { 265 return FileSystemAccess.class; 266 } 267 268 @Override 269 public Class[] getServiceDependencies() { 270 return new Class[]{Instrumentation.class, Scheduler.class}; 271 } 272 273 protected UserGroupInformation getUGI(String user) throws IOException { 274 return UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser()); 275 } 276 277 protected void setRequiredServiceHadoopConf(Configuration conf) { 278 conf.set("fs.hdfs.impl.disable.cache", "true"); 279 } 280 281 private static final String HTTPFS_FS_USER = "httpfs.fs.user"; 282 283 protected FileSystem createFileSystem(Configuration namenodeConf) 284 throws IOException { 285 String user = UserGroupInformation.getCurrentUser().getShortUserName(); 286 CachedFileSystem newCachedFS = new CachedFileSystem(purgeTimeout); 287 CachedFileSystem cachedFS = fsCache.putIfAbsent(user, newCachedFS); 288 if (cachedFS == null) { 289 cachedFS = newCachedFS; 290 } 291 Configuration conf = new Configuration(namenodeConf); 292 conf.set(HTTPFS_FS_USER, user); 293 return cachedFS.getFileSytem(conf); 294 } 295 296 protected void closeFileSystem(FileSystem fs) throws IOException { 297 if (fsCache.containsKey(fs.getConf().get(HTTPFS_FS_USER))) { 298 fsCache.get(fs.getConf().get(HTTPFS_FS_USER)).release(); 299 } 300 } 301 302 protected void validateNamenode(String namenode) throws FileSystemAccessException { 303 if (nameNodeWhitelist.size() > 0 && !nameNodeWhitelist.contains("*")) { 304 if (!nameNodeWhitelist.contains( 305 StringUtils.toLowerCase(namenode))) { 306 throw new FileSystemAccessException(FileSystemAccessException.ERROR.H05, namenode, "not in whitelist"); 307 } 308 } 309 } 310 311 protected void checkNameNodeHealth(FileSystem fileSystem) throws FileSystemAccessException { 312 } 313 314 @Override 315 public <T> T execute(String user, final Configuration conf, final FileSystemExecutor<T> executor) 316 throws FileSystemAccessException { 317 Check.notEmpty(user, "user"); 318 Check.notNull(conf, "conf"); 319 Check.notNull(executor, "executor"); 320 if (!conf.getBoolean(FILE_SYSTEM_SERVICE_CREATED, false)) { 321 throw new FileSystemAccessException(FileSystemAccessException.ERROR.H04); 322 } 323 if (conf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY) == null || 324 conf.getTrimmed(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY).length() == 0) { 325 throw new FileSystemAccessException(FileSystemAccessException.ERROR.H06, 326 CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY); 327 } 328 try { 329 validateNamenode( 330 new URI(conf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY)). 331 getAuthority()); 332 UserGroupInformation ugi = getUGI(user); 333 return ugi.doAs(new PrivilegedExceptionAction<T>() { 334 @Override 335 public T run() throws Exception { 336 FileSystem fs = createFileSystem(conf); 337 Instrumentation instrumentation = getServer().get(Instrumentation.class); 338 Instrumentation.Cron cron = instrumentation.createCron(); 339 try { 340 checkNameNodeHealth(fs); 341 cron.start(); 342 return executor.execute(fs); 343 } finally { 344 cron.stop(); 345 instrumentation.addCron(INSTRUMENTATION_GROUP, executor.getClass().getSimpleName(), cron); 346 closeFileSystem(fs); 347 } 348 } 349 }); 350 } catch (FileSystemAccessException ex) { 351 throw ex; 352 } catch (Exception ex) { 353 throw new FileSystemAccessException(FileSystemAccessException.ERROR.H03, ex); 354 } 355 } 356 357 public FileSystem createFileSystemInternal(String user, final Configuration conf) 358 throws IOException, FileSystemAccessException { 359 Check.notEmpty(user, "user"); 360 Check.notNull(conf, "conf"); 361 if (!conf.getBoolean(FILE_SYSTEM_SERVICE_CREATED, false)) { 362 throw new FileSystemAccessException(FileSystemAccessException.ERROR.H04); 363 } 364 try { 365 validateNamenode( 366 new URI(conf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY)).getAuthority()); 367 UserGroupInformation ugi = getUGI(user); 368 return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() { 369 @Override 370 public FileSystem run() throws Exception { 371 return createFileSystem(conf); 372 } 373 }); 374 } catch (IOException ex) { 375 throw ex; 376 } catch (FileSystemAccessException ex) { 377 throw ex; 378 } catch (Exception ex) { 379 throw new FileSystemAccessException(FileSystemAccessException.ERROR.H08, ex.getMessage(), ex); 380 } 381 } 382 383 @Override 384 public FileSystem createFileSystem(String user, final Configuration conf) throws IOException, 385 FileSystemAccessException { 386 unmanagedFileSystems.incrementAndGet(); 387 return createFileSystemInternal(user, conf); 388 } 389 390 @Override 391 public void releaseFileSystem(FileSystem fs) throws IOException { 392 unmanagedFileSystems.decrementAndGet(); 393 closeFileSystem(fs); 394 } 395 396 @Override 397 public Configuration getFileSystemConfiguration() { 398 Configuration conf = new Configuration(true); 399 ConfigurationUtils.copy(serviceHadoopConf, conf); 400 conf.setBoolean(FILE_SYSTEM_SERVICE_CREATED, true); 401 402 // Force-clear server-side umask to make HttpFS match WebHDFS behavior 403 conf.set(FsPermission.UMASK_LABEL, "000"); 404 405 return conf; 406 } 407 408}