namenode/snapshot/SnapshotFSImageFormat.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hdfs.server.namenode.snapshot;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormat;
import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeAttributes;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectoryAttributes;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.INodeFileAttributes;
import org.apache.hadoop.hdfs.server.namenode.INodeReference;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiffList;
import org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff;
import org.apache.hadoop.hdfs.util.Diff.ListType;
import org.apache.hadoop.hdfs.util.ReadOnlyList;

import com.google.common.base.Preconditions;

/**
 * A helper class defining static methods for reading/writing snapshot related
 * information from/to FSImage.
 */
public class SnapshotFSImageFormat {
  /**
   * Save snapshots and snapshot quota for a snapshottable directory.
   * @param current The directory that the snapshots belongs to.
   * @param out The {@link DataOutput} to write.
   * @throws IOException
   */
  public static void saveSnapshots(INodeDirectory current, DataOutput out)
      throws IOException {
    DirectorySnapshottableFeature sf = current.getDirectorySnapshottableFeature();
    Preconditions.checkArgument(sf != null);
    // list of snapshots in snapshotsByNames
    ReadOnlyList<Snapshot> snapshots = sf.getSnapshotList();
    out.writeInt(snapshots.size());
    for (Snapshot s : snapshots) {
      // write the snapshot id
      out.writeInt(s.getId());
    }
    // snapshot quota
    out.writeInt(sf.getSnapshotQuota());
  }

  /**
   * Save SnapshotDiff list for an INodeDirectoryWithSnapshot.
   * @param sNode The directory that the SnapshotDiff list belongs to.
   * @param out The {@link DataOutput} to write.
   */
  private static <N extends INode, A extends INodeAttributes, D extends AbstractINodeDiff<N, A, D>>
      void saveINodeDiffs(final AbstractINodeDiffList<N, A, D> diffs,
      final DataOutput out, ReferenceMap referenceMap) throws IOException {
    // Record the diffs in reversed order, so that we can find the correct
    // reference for INodes in the created list when loading the FSImage
    if (diffs == null) {
      out.writeInt(-1); // no diffs
    } else {
      final List<D> list = diffs.asList();
      final int size = list.size();
      out.writeInt(size);
      for (int i = size - 1; i >= 0; i--) {
        list.get(i).write(out, referenceMap);
      }
    }
  }

  public static void saveDirectoryDiffList(final INodeDirectory dir,
      final DataOutput out, final ReferenceMap referenceMap
      ) throws IOException {
    saveINodeDiffs(dir.getDiffs(), out, referenceMap);
  }

  public static void saveFileDiffList(final INodeFile file,
      final DataOutput out) throws IOException {
    saveINodeDiffs(file.getDiffs(), out, null);
  }

  public static FileDiffList loadFileDiffList(DataInput in,
      FSImageFormat.Loader loader) throws IOException {
    final int size = in.readInt();
    if (size == -1) {
      return null;
    } else {
      final FileDiffList diffs = new FileDiffList();
      FileDiff posterior = null;
      for(int i = 0; i < size; i++) {
        final FileDiff d = loadFileDiff(posterior, in, loader);
        diffs.addFirst(d);
        posterior = d;
      }
      return diffs;
    }
  }

  private static FileDiff loadFileDiff(FileDiff posterior, DataInput in,
      FSImageFormat.Loader loader) throws IOException {
    // 1. Read the id of the Snapshot root to identify the Snapshot
    final Snapshot snapshot = loader.getSnapshot(in);

    // 2. Load file size
    final long fileSize = in.readLong();

    // 3. Load snapshotINode
    final INodeFileAttributes snapshotINode = in.readBoolean()?
        loader.loadINodeFileAttributes(in): null;

    return new FileDiff(snapshot.getId(), snapshotINode, posterior, fileSize);
  }

  /**
   * Load a node stored in the created list from fsimage.
   * @param createdNodeName The name of the created node.
   * @param parent The directory that the created list belongs to.
   * @return The created node.
   */
  public static INode loadCreated(byte[] createdNodeName,
      INodeDirectory parent) throws IOException {
    // the INode in the created list should be a reference to another INode
    // in posterior SnapshotDiffs or one of the current children
    for (DirectoryDiff postDiff : parent.getDiffs()) {
      final INode d = postDiff.getChildrenDiff().search(ListType.DELETED,
          createdNodeName);
      if (d != null) {
        return d;
      } // else go to the next SnapshotDiff
    }
    // use the current child
    INode currentChild = parent.getChild(createdNodeName,
        Snapshot.CURRENT_STATE_ID);
    if (currentChild == null) {
      throw new IOException("Cannot find an INode associated with the INode "
          + DFSUtil.bytes2String(createdNodeName)
          + " in created list while loading FSImage.");
    }
    return currentChild;
  }

  /**
   * Load the created list from fsimage.
   * @param parent The directory that the created list belongs to.
   * @param in The {@link DataInput} to read.
   * @return The created list.
   */
  private static List<INode> loadCreatedList(INodeDirectory parent,
      DataInput in) throws IOException {
    // read the size of the created list
    int createdSize = in.readInt();
    List<INode> createdList = new ArrayList<INode>(createdSize);
    for (int i = 0; i < createdSize; i++) {
      byte[] createdNodeName = FSImageSerialization.readLocalName(in);
      INode created = loadCreated(createdNodeName, parent);
      createdList.add(created);
    }
    return createdList;
  }

  /**
   * Load the deleted list from the fsimage.
   *
   * @param parent The directory that the deleted list belongs to.
   * @param createdList The created list associated with the deleted list in
   *                    the same Diff.
   * @param in The {@link DataInput} to read.
   * @param loader The {@link Loader} instance.
   * @return The deleted list.
   */
  private static List<INode> loadDeletedList(INodeDirectory parent,
      List<INode> createdList, DataInput in, FSImageFormat.Loader loader)
      throws IOException {
    int deletedSize = in.readInt();
    List<INode> deletedList = new ArrayList<INode>(deletedSize);
    for (int i = 0; i < deletedSize; i++) {
      final INode deleted = loader.loadINodeWithLocalName(true, in, true);
      deletedList.add(deleted);
      // set parent: the parent field of an INode in the deleted list is not
      // useful, but set the parent here to be consistent with the original
      // fsdir tree.
      deleted.setParent(parent);
      if (deleted.isFile()) {
        loader.updateBlocksMap(deleted.asFile());
      }
    }
    return deletedList;
  }

  /**
   * Load snapshots and snapshotQuota for a Snapshottable directory.
   *
   * @param snapshottableParent
   *          The snapshottable directory for loading.
   * @param numSnapshots
   *          The number of snapshots that the directory has.
   * @param loader
   *          The loader
   */
  public static void loadSnapshotList(INodeDirectory snapshottableParent,
      int numSnapshots, DataInput in, FSImageFormat.Loader loader)
      throws IOException {
    DirectorySnapshottableFeature sf = snapshottableParent
        .getDirectorySnapshottableFeature();
    Preconditions.checkArgument(sf != null);
    for (int i = 0; i < numSnapshots; i++) {
      // read snapshots
      final Snapshot s = loader.getSnapshot(in);
      s.getRoot().setParent(snapshottableParent);
      sf.addSnapshot(s);
    }
    int snapshotQuota = in.readInt();
    snapshottableParent.setSnapshotQuota(snapshotQuota);
  }

  /**
   * Load the {@link SnapshotDiff} list for the INodeDirectoryWithSnapshot
   * directory.
   *
   * @param dir
   *          The snapshottable directory for loading.
   * @param in
   *          The {@link DataInput} instance to read.
   * @param loader
   *          The loader
   */
  public static void loadDirectoryDiffList(INodeDirectory dir,
      DataInput in, FSImageFormat.Loader loader) throws IOException {
    final int size = in.readInt();
    if (dir.isWithSnapshot()) {
      DirectoryDiffList diffs = dir.getDiffs();
      for (int i = 0; i < size; i++) {
        diffs.addFirst(loadDirectoryDiff(dir, in, loader));
      }
    }
  }

  /**
   * Load the snapshotINode field of {@link AbstractINodeDiff}.
   * @param snapshot The Snapshot associated with the {@link AbstractINodeDiff}.
   * @param in The {@link DataInput} to read.
   * @param loader The {@link Loader} instance that this loading procedure is
   *               using.
   * @return The snapshotINode.
   */
  private static INodeDirectoryAttributes loadSnapshotINodeInDirectoryDiff(
      Snapshot snapshot, DataInput in, FSImageFormat.Loader loader)
      throws IOException {
    // read the boolean indicating whether snapshotINode == Snapshot.Root
    boolean useRoot = in.readBoolean();
    if (useRoot) {
      return snapshot.getRoot();
    } else {
      // another boolean is used to indicate whether snapshotINode is non-null
      return in.readBoolean()? loader.loadINodeDirectoryAttributes(in): null;
    }
  }

  /**
   * Load {@link DirectoryDiff} from fsimage.
   * @param parent The directory that the SnapshotDiff belongs to.
   * @param in The {@link DataInput} instance to read.
   * @param loader The {@link Loader} instance that this loading procedure is
   *               using.
   * @return A {@link DirectoryDiff}.
   */
  private static DirectoryDiff loadDirectoryDiff(INodeDirectory parent,
      DataInput in, FSImageFormat.Loader loader) throws IOException {
    // 1. Read the full path of the Snapshot root to identify the Snapshot
    final Snapshot snapshot = loader.getSnapshot(in);

    // 2. Load DirectoryDiff#childrenSize
    int childrenSize = in.readInt();

    // 3. Load DirectoryDiff#snapshotINode
    INodeDirectoryAttributes snapshotINode = loadSnapshotINodeInDirectoryDiff(
        snapshot, in, loader);

    // 4. Load the created list in SnapshotDiff#Diff
    List<INode> createdList = loadCreatedList(parent, in);

    // 5. Load the deleted list in SnapshotDiff#Diff
    List<INode> deletedList = loadDeletedList(parent, createdList, in, loader);

    // 6. Compose the SnapshotDiff
    List<DirectoryDiff> diffs = parent.getDiffs().asList();
    DirectoryDiff sdiff = new DirectoryDiff(snapshot.getId(), snapshotINode,
        diffs.isEmpty() ? null : diffs.get(0), childrenSize, createdList,
        deletedList, snapshotINode == snapshot.getRoot());
    return sdiff;
  }


  /** A reference map for fsimage serialization. */
  public static class ReferenceMap {
    /**
     * Used to indicate whether the reference node itself has been saved
     */
    private final Map<Long, INodeReference.WithCount> referenceMap
        = new HashMap<Long, INodeReference.WithCount>();
    /**
     * Used to record whether the subtree of the reference node has been saved
     */
    private final Map<Long, Long> dirMap = new HashMap<Long, Long>();

    public void writeINodeReferenceWithCount(
        INodeReference.WithCount withCount, DataOutput out,
        boolean writeUnderConstruction) throws IOException {
      final INode referred = withCount.getReferredINode();
      final long id = withCount.getId();
      final boolean firstReferred = !referenceMap.containsKey(id);
      out.writeBoolean(firstReferred);

      if (firstReferred) {
        FSImageSerialization.saveINode2Image(referred, out,
            writeUnderConstruction, this);
        referenceMap.put(id, withCount);
      } else {
        out.writeLong(id);
      }
    }

    public boolean toProcessSubtree(long id) {
      if (dirMap.containsKey(id)) {
        return false;
      } else {
        dirMap.put(id, id);
        return true;
      }
    }

    public INodeReference.WithCount loadINodeReferenceWithCount(
        boolean isSnapshotINode, DataInput in, FSImageFormat.Loader loader
        ) throws IOException {
      final boolean firstReferred = in.readBoolean();

      final INodeReference.WithCount withCount;
      if (firstReferred) {
        final INode referred = loader.loadINodeWithLocalName(isSnapshotINode,
            in, true);
        withCount = new INodeReference.WithCount(null, referred);
        referenceMap.put(withCount.getId(), withCount);
      } else {
        final long id = in.readLong();
        withCount = referenceMap.get(id);
      }
      return withCount;
    }
  }
}