1 /**
2  * Licensed to the Apache Software Foundation (ASF) under one
3  * or more contributor license agreements.  See the NOTICE file
4  * distributed with this work for additional information
5  * regarding copyright ownership.  The ASF licenses this file
6  * to you under the Apache License, Version 2.0 (the
7  * "License"); you may not use this file except in compliance
8  * with the License.  You may obtain a copy of the License at
9  *
10  *     http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing, software
13  * distributed under the License is distributed on an "AS IS" BASIS,
14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  * See the License for the specific language governing permissions and
16  * limitations under the License.
17  */
18 package org.apache.hadoop.hdfs.server.namenode.ha;
19 
20 
21 import static org.junit.Assert.assertEquals;
22 
23 import java.io.IOException;
24 
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.fs.ContentSummary;
27 import org.apache.hadoop.fs.FSDataOutputStream;
28 import org.apache.hadoop.fs.FileSystem;
29 import org.apache.hadoop.fs.Path;
30 import org.apache.hadoop.hdfs.DFSConfigKeys;
31 import org.apache.hadoop.hdfs.DFSTestUtil;
32 import org.apache.hadoop.hdfs.DistributedFileSystem;
33 import org.apache.hadoop.hdfs.HAUtil;
34 import org.apache.hadoop.hdfs.MiniDFSCluster;
35 import org.apache.hadoop.hdfs.MiniDFSNNTopology;
36 import org.apache.hadoop.hdfs.server.namenode.NameNode;
37 import org.apache.hadoop.io.IOUtils;
38 import org.apache.hadoop.ipc.StandbyException;
39 import org.junit.After;
40 import org.junit.Before;
41 import org.junit.Test;
42 
43 public class TestQuotasWithHA {
44   private static final Path TEST_DIR = new Path("/test");
45   private static final Path TEST_FILE = new Path(TEST_DIR, "file");
46   private static final String TEST_DIR_STR = TEST_DIR.toUri().getPath();
47 
48   private static final long NS_QUOTA = 10000;
49   private static final long DS_QUOTA = 10000;
50   private static final long BLOCK_SIZE = 1024; // 1KB blocks
51 
52   private MiniDFSCluster cluster;
53   private NameNode nn0;
54   private NameNode nn1;
55   private FileSystem fs;
56 
57   @Before
setupCluster()58   public void setupCluster() throws Exception {
59     Configuration conf = new Configuration();
60     conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
61     conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
62     conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
63     HAUtil.setAllowStandbyReads(conf, true);
64 
65     cluster = new MiniDFSCluster.Builder(conf)
66       .nnTopology(MiniDFSNNTopology.simpleHATopology())
67       .numDataNodes(1)
68       .waitSafeMode(false)
69       .build();
70     cluster.waitActive();
71 
72     nn0 = cluster.getNameNode(0);
73     nn1 = cluster.getNameNode(1);
74     fs = HATestUtil.configureFailoverFs(cluster, conf);
75 
76     cluster.transitionToActive(0);
77   }
78 
79   @After
shutdownCluster()80   public void shutdownCluster() throws IOException {
81     if (cluster != null) {
82       cluster.shutdown();
83     }
84   }
85 
86   /**
87    * Test that quotas are properly tracked by the standby through
88    * create, append, delete.
89    */
90   @Test(timeout=60000)
testQuotasTrackedOnStandby()91   public void testQuotasTrackedOnStandby() throws Exception {
92     fs.mkdirs(TEST_DIR);
93     DistributedFileSystem dfs = (DistributedFileSystem)fs;
94     dfs.setQuota(TEST_DIR, NS_QUOTA, DS_QUOTA);
95     long expectedSize = 3 * BLOCK_SIZE + BLOCK_SIZE/2;
96     DFSTestUtil.createFile(fs, TEST_FILE, expectedSize, (short)1, 1L);
97 
98     HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
99     ContentSummary cs = nn1.getRpcServer().getContentSummary(TEST_DIR_STR);
100     assertEquals(NS_QUOTA, cs.getQuota());
101     assertEquals(DS_QUOTA, cs.getSpaceQuota());
102     assertEquals(expectedSize, cs.getSpaceConsumed());
103     assertEquals(1, cs.getDirectoryCount());
104     assertEquals(1, cs.getFileCount());
105 
106     // Append to the file and make sure quota is updated correctly.
107     FSDataOutputStream stm = fs.append(TEST_FILE);
108     try {
109       byte[] data = new byte[(int) (BLOCK_SIZE * 3 / 2)];
110       stm.write(data);
111       expectedSize += data.length;
112     } finally {
113       IOUtils.closeStream(stm);
114     }
115 
116     HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
117     cs = nn1.getRpcServer().getContentSummary(TEST_DIR_STR);
118     assertEquals(NS_QUOTA, cs.getQuota());
119     assertEquals(DS_QUOTA, cs.getSpaceQuota());
120     assertEquals(expectedSize, cs.getSpaceConsumed());
121     assertEquals(1, cs.getDirectoryCount());
122     assertEquals(1, cs.getFileCount());
123 
124 
125     fs.delete(TEST_FILE, true);
126     expectedSize = 0;
127     HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
128     cs = nn1.getRpcServer().getContentSummary(TEST_DIR_STR);
129     assertEquals(NS_QUOTA, cs.getQuota());
130     assertEquals(DS_QUOTA, cs.getSpaceQuota());
131     assertEquals(expectedSize, cs.getSpaceConsumed());
132     assertEquals(1, cs.getDirectoryCount());
133     assertEquals(0, cs.getFileCount());
134   }
135 
136   /**
137    * Test that getContentSummary on Standby should should throw standby
138    * exception.
139    */
140   @Test(expected = StandbyException.class)
testgetContentSummaryOnStandby()141   public void testgetContentSummaryOnStandby() throws Exception {
142     Configuration nn1conf =cluster.getConfiguration(1);
143     // just reset the standby reads to default i.e False on standby.
144     HAUtil.setAllowStandbyReads(nn1conf, false);
145     cluster.restartNameNode(1);
146     cluster.getNameNodeRpc(1).getContentSummary("/");
147   }
148 }
149