001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.lib.service.hadoop;
020
021import org.apache.hadoop.classification.InterfaceAudience;
022import org.apache.hadoop.conf.Configuration;
023import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
024import org.apache.hadoop.fs.FileSystem;
025import org.apache.hadoop.fs.Path;
026import org.apache.hadoop.fs.permission.FsPermission;
027import org.apache.hadoop.lib.server.BaseService;
028import org.apache.hadoop.lib.server.ServiceException;
029import org.apache.hadoop.lib.service.FileSystemAccess;
030import org.apache.hadoop.lib.service.FileSystemAccessException;
031import org.apache.hadoop.lib.service.Instrumentation;
032import org.apache.hadoop.lib.service.Scheduler;
033import org.apache.hadoop.lib.util.Check;
034import org.apache.hadoop.lib.util.ConfigurationUtils;
035import org.apache.hadoop.security.UserGroupInformation;
036import org.apache.hadoop.util.StringUtils;
037import org.apache.hadoop.util.VersionInfo;
038import org.slf4j.Logger;
039import org.slf4j.LoggerFactory;
040
041import java.io.File;
042import java.io.IOException;
043import java.net.URI;
044import java.security.PrivilegedExceptionAction;
045import java.util.Collection;
046import java.util.HashSet;
047import java.util.Map;
048import java.util.Set;
049import java.util.concurrent.ConcurrentHashMap;
050import java.util.concurrent.TimeUnit;
051import java.util.concurrent.atomic.AtomicInteger;
052
053@InterfaceAudience.Private
054public class FileSystemAccessService extends BaseService implements FileSystemAccess {
055  private static final Logger LOG = LoggerFactory.getLogger(FileSystemAccessService.class);
056
057  public static final String PREFIX = "hadoop";
058
059  private static final String INSTRUMENTATION_GROUP = "hadoop";
060
061  public static final String AUTHENTICATION_TYPE = "authentication.type";
062  public static final String KERBEROS_KEYTAB = "authentication.kerberos.keytab";
063  public static final String KERBEROS_PRINCIPAL = "authentication.kerberos.principal";
064  public static final String FS_CACHE_PURGE_FREQUENCY = "filesystem.cache.purge.frequency";
065  public static final String FS_CACHE_PURGE_TIMEOUT = "filesystem.cache.purge.timeout";
066
067  public static final String NAME_NODE_WHITELIST = "name.node.whitelist";
068
069  public static final String HADOOP_CONF_DIR = "config.dir";
070
071  private static final String[] HADOOP_CONF_FILES = {"core-site.xml", "hdfs-site.xml"};
072
073  private static final String FILE_SYSTEM_SERVICE_CREATED = "FileSystemAccessService.created";
074
075  private static class CachedFileSystem {
076    private FileSystem fs;
077    private long lastUse;
078    private long timeout;
079    private int count;
080
081    public CachedFileSystem(long timeout) {
082      this.timeout = timeout;
083      lastUse = -1;
084      count = 0;
085    }
086
087    synchronized FileSystem getFileSytem(Configuration conf)
088      throws IOException {
089      if (fs == null) {
090        fs = FileSystem.get(conf);
091      }
092      lastUse = -1;
093      count++;
094      return fs;
095    }
096
097    synchronized void release() throws IOException {
098      count--;
099      if (count == 0) {
100        if (timeout == 0) {
101          fs.close();
102          fs = null;
103          lastUse = -1;
104        }
105        else {
106          lastUse = System.currentTimeMillis();
107        }
108      }
109    }
110
111    // to avoid race conditions in the map cache adding removing entries
112    // an entry in the cache remains forever, it just closes/opens filesystems
113    // based on their utilization. Worse case scenario, the penalty we'll
114    // pay is that the amount of entries in the cache will be the total
115    // number of users in HDFS (which seems a resonable overhead).
116    synchronized boolean purgeIfIdle() throws IOException {
117      boolean ret = false;
118      if (count == 0 && lastUse != -1 &&
119          (System.currentTimeMillis() - lastUse) > timeout) {
120        fs.close();
121        fs = null;
122        lastUse = -1;
123        ret = true;
124      }
125      return ret;
126    }
127
128  }
129
130  public FileSystemAccessService() {
131    super(PREFIX);
132  }
133
134  private Collection<String> nameNodeWhitelist;
135
136  Configuration serviceHadoopConf;
137
138  private AtomicInteger unmanagedFileSystems = new AtomicInteger();
139
140  private ConcurrentHashMap<String, CachedFileSystem> fsCache =
141    new ConcurrentHashMap<String, CachedFileSystem>();
142
143  private long purgeTimeout;
144
145  @Override
146  protected void init() throws ServiceException {
147    LOG.info("Using FileSystemAccess JARs version [{}]", VersionInfo.getVersion());
148    String security = getServiceConfig().get(AUTHENTICATION_TYPE, "simple").trim();
149    if (security.equals("kerberos")) {
150      String defaultName = getServer().getName();
151      String keytab = System.getProperty("user.home") + "/" + defaultName + ".keytab";
152      keytab = getServiceConfig().get(KERBEROS_KEYTAB, keytab).trim();
153      if (keytab.length() == 0) {
154        throw new ServiceException(FileSystemAccessException.ERROR.H01, KERBEROS_KEYTAB);
155      }
156      String principal = defaultName + "/localhost@LOCALHOST";
157      principal = getServiceConfig().get(KERBEROS_PRINCIPAL, principal).trim();
158      if (principal.length() == 0) {
159        throw new ServiceException(FileSystemAccessException.ERROR.H01, KERBEROS_PRINCIPAL);
160      }
161      Configuration conf = new Configuration();
162      conf.set("hadoop.security.authentication", "kerberos");
163      UserGroupInformation.setConfiguration(conf);
164      try {
165        UserGroupInformation.loginUserFromKeytab(principal, keytab);
166      } catch (IOException ex) {
167        throw new ServiceException(FileSystemAccessException.ERROR.H02, ex.getMessage(), ex);
168      }
169      LOG.info("Using FileSystemAccess Kerberos authentication, principal [{}] keytab [{}]", principal, keytab);
170    } else if (security.equals("simple")) {
171      Configuration conf = new Configuration();
172      conf.set("hadoop.security.authentication", "simple");
173      UserGroupInformation.setConfiguration(conf);
174      LOG.info("Using FileSystemAccess simple/pseudo authentication, principal [{}]", System.getProperty("user.name"));
175    } else {
176      throw new ServiceException(FileSystemAccessException.ERROR.H09, security);
177    }
178
179    String hadoopConfDirProp = getServiceConfig().get(HADOOP_CONF_DIR, getServer().getConfigDir());
180    File hadoopConfDir = new File(hadoopConfDirProp).getAbsoluteFile();
181    if (!hadoopConfDir.exists()) {
182      hadoopConfDir = new File(getServer().getConfigDir()).getAbsoluteFile();
183    }
184    if (!hadoopConfDir.exists()) {
185      throw new ServiceException(FileSystemAccessException.ERROR.H10, hadoopConfDir);
186    }
187    try {
188      serviceHadoopConf = loadHadoopConf(hadoopConfDir);
189    } catch (IOException ex) {
190      throw new ServiceException(FileSystemAccessException.ERROR.H11, ex.toString(), ex);
191    }
192
193    LOG.debug("FileSystemAccess FileSystem configuration:");
194    for (Map.Entry entry : serviceHadoopConf) {
195      LOG.debug("  {} = {}", entry.getKey(), entry.getValue());
196    }
197    setRequiredServiceHadoopConf(serviceHadoopConf);
198
199    nameNodeWhitelist = toLowerCase(getServiceConfig().getTrimmedStringCollection(NAME_NODE_WHITELIST));
200  }
201
202  private Configuration loadHadoopConf(File dir) throws IOException {
203    Configuration hadoopConf = new Configuration(false);
204    for (String file : HADOOP_CONF_FILES) {
205      File f = new File(dir, file);
206      if (f.exists()) {
207        hadoopConf.addResource(new Path(f.getAbsolutePath()));
208      }
209    }
210    return hadoopConf;
211  }
212
213  @Override
214  public void postInit() throws ServiceException {
215    super.postInit();
216    Instrumentation instrumentation = getServer().get(Instrumentation.class);
217    instrumentation.addVariable(INSTRUMENTATION_GROUP, "unmanaged.fs", new Instrumentation.Variable<Integer>() {
218      @Override
219      public Integer getValue() {
220        return unmanagedFileSystems.get();
221      }
222    });
223    instrumentation.addSampler(INSTRUMENTATION_GROUP, "unmanaged.fs", 60, new Instrumentation.Variable<Long>() {
224      @Override
225      public Long getValue() {
226        return (long) unmanagedFileSystems.get();
227      }
228    });
229    Scheduler scheduler = getServer().get(Scheduler.class);
230    int purgeInterval = getServiceConfig().getInt(FS_CACHE_PURGE_FREQUENCY, 60);
231    purgeTimeout = getServiceConfig().getLong(FS_CACHE_PURGE_TIMEOUT, 60);
232    purgeTimeout = (purgeTimeout > 0) ? purgeTimeout : 0;
233    if (purgeTimeout > 0) {
234      scheduler.schedule(new FileSystemCachePurger(),
235                         purgeInterval, purgeInterval, TimeUnit.SECONDS);
236    }
237  }
238
239  private class FileSystemCachePurger implements Runnable {
240
241    @Override
242    public void run() {
243      int count = 0;
244      for (CachedFileSystem cacheFs : fsCache.values()) {
245        try {
246          count += cacheFs.purgeIfIdle() ? 1 : 0;
247        } catch (Throwable ex) {
248          LOG.warn("Error while purging filesystem, " + ex.toString(), ex);
249        }
250      }
251      LOG.debug("Purged [{}} filesystem instances", count);
252    }
253  }
254
255  private Set<String> toLowerCase(Collection<String> collection) {
256    Set<String> set = new HashSet<String>();
257    for (String value : collection) {
258      set.add(StringUtils.toLowerCase(value));
259    }
260    return set;
261  }
262
263  @Override
264  public Class getInterface() {
265    return FileSystemAccess.class;
266  }
267
268  @Override
269  public Class[] getServiceDependencies() {
270    return new Class[]{Instrumentation.class, Scheduler.class};
271  }
272
273  protected UserGroupInformation getUGI(String user) throws IOException {
274    return UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser());
275  }
276
277  protected void setRequiredServiceHadoopConf(Configuration conf) {
278    conf.set("fs.hdfs.impl.disable.cache", "true");
279  }
280
281  private static final String HTTPFS_FS_USER = "httpfs.fs.user";
282
283  protected FileSystem createFileSystem(Configuration namenodeConf)
284    throws IOException {
285    String user = UserGroupInformation.getCurrentUser().getShortUserName();
286    CachedFileSystem newCachedFS = new CachedFileSystem(purgeTimeout);
287    CachedFileSystem cachedFS = fsCache.putIfAbsent(user, newCachedFS);
288    if (cachedFS == null) {
289      cachedFS = newCachedFS;
290    }
291    Configuration conf = new Configuration(namenodeConf);
292    conf.set(HTTPFS_FS_USER, user);
293    return cachedFS.getFileSytem(conf);
294  }
295
296  protected void closeFileSystem(FileSystem fs) throws IOException {
297    if (fsCache.containsKey(fs.getConf().get(HTTPFS_FS_USER))) {
298      fsCache.get(fs.getConf().get(HTTPFS_FS_USER)).release();
299    }
300  }
301
302  protected void validateNamenode(String namenode) throws FileSystemAccessException {
303    if (nameNodeWhitelist.size() > 0 && !nameNodeWhitelist.contains("*")) {
304      if (!nameNodeWhitelist.contains(
305          StringUtils.toLowerCase(namenode))) {
306        throw new FileSystemAccessException(FileSystemAccessException.ERROR.H05, namenode, "not in whitelist");
307      }
308    }
309  }
310
311  protected void checkNameNodeHealth(FileSystem fileSystem) throws FileSystemAccessException {
312  }
313
314  @Override
315  public <T> T execute(String user, final Configuration conf, final FileSystemExecutor<T> executor)
316    throws FileSystemAccessException {
317    Check.notEmpty(user, "user");
318    Check.notNull(conf, "conf");
319    Check.notNull(executor, "executor");
320    if (!conf.getBoolean(FILE_SYSTEM_SERVICE_CREATED, false)) {
321      throw new FileSystemAccessException(FileSystemAccessException.ERROR.H04);
322    }
323    if (conf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY) == null ||
324        conf.getTrimmed(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY).length() == 0) {
325      throw new FileSystemAccessException(FileSystemAccessException.ERROR.H06,
326                                          CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY);
327    }
328    try {
329      validateNamenode(
330        new URI(conf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY)).
331          getAuthority());
332      UserGroupInformation ugi = getUGI(user);
333      return ugi.doAs(new PrivilegedExceptionAction<T>() {
334        @Override
335        public T run() throws Exception {
336          FileSystem fs = createFileSystem(conf);
337          Instrumentation instrumentation = getServer().get(Instrumentation.class);
338          Instrumentation.Cron cron = instrumentation.createCron();
339          try {
340            checkNameNodeHealth(fs);
341            cron.start();
342            return executor.execute(fs);
343          } finally {
344            cron.stop();
345            instrumentation.addCron(INSTRUMENTATION_GROUP, executor.getClass().getSimpleName(), cron);
346            closeFileSystem(fs);
347          }
348        }
349      });
350    } catch (FileSystemAccessException ex) {
351      throw ex;
352    } catch (Exception ex) {
353      throw new FileSystemAccessException(FileSystemAccessException.ERROR.H03, ex);
354    }
355  }
356
357  public FileSystem createFileSystemInternal(String user, final Configuration conf)
358    throws IOException, FileSystemAccessException {
359    Check.notEmpty(user, "user");
360    Check.notNull(conf, "conf");
361    if (!conf.getBoolean(FILE_SYSTEM_SERVICE_CREATED, false)) {
362      throw new FileSystemAccessException(FileSystemAccessException.ERROR.H04);
363    }
364    try {
365      validateNamenode(
366        new URI(conf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY)).getAuthority());
367      UserGroupInformation ugi = getUGI(user);
368      return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
369        @Override
370        public FileSystem run() throws Exception {
371          return createFileSystem(conf);
372        }
373      });
374    } catch (IOException ex) {
375      throw ex;
376    } catch (FileSystemAccessException ex) {
377      throw ex;
378    } catch (Exception ex) {
379      throw new FileSystemAccessException(FileSystemAccessException.ERROR.H08, ex.getMessage(), ex);
380    }
381  }
382
383  @Override
384  public FileSystem createFileSystem(String user, final Configuration conf) throws IOException,
385    FileSystemAccessException {
386    unmanagedFileSystems.incrementAndGet();
387    return createFileSystemInternal(user, conf);
388  }
389
390  @Override
391  public void releaseFileSystem(FileSystem fs) throws IOException {
392    unmanagedFileSystems.decrementAndGet();
393    closeFileSystem(fs);
394  }
395
396  @Override
397  public Configuration getFileSystemConfiguration() {
398    Configuration conf = new Configuration(true);
399    ConfigurationUtils.copy(serviceHadoopConf, conf);
400    conf.setBoolean(FILE_SYSTEM_SERVICE_CREATED, true);
401
402    // Force-clear server-side umask to make HttpFS match WebHDFS behavior
403    conf.set(FsPermission.UMASK_LABEL, "000");
404
405    return conf;
406  }
407
408}