1 /**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements.  See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership.  The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License.  You may obtain a copy of the License at
9 *
10 *     http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18 
19 package org.apache.hadoop.yarn.server.nodemanager.webapp;
20 
21 import static org.mockito.Mockito.mock;
22 import static org.mockito.Mockito.when;
23 
24 import java.io.File;
25 import java.io.FileWriter;
26 import java.io.IOException;
27 import java.io.Writer;
28 
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.FileUtil;
31 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
32 import org.apache.hadoop.yarn.api.records.ApplicationId;
33 import org.apache.hadoop.yarn.api.records.ContainerId;
34 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
35 import org.apache.hadoop.yarn.api.records.Token;
36 import org.apache.hadoop.yarn.conf.YarnConfiguration;
37 import org.apache.hadoop.yarn.event.AsyncDispatcher;
38 import org.apache.hadoop.yarn.event.Dispatcher;
39 import org.apache.hadoop.yarn.exceptions.YarnException;
40 import org.apache.hadoop.yarn.factories.RecordFactory;
41 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
42 import org.apache.hadoop.yarn.server.nodemanager.Context;
43 import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
44 import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
45 import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
46 import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
47 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
48 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
49 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
50 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
51 import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
52 import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
53 import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
54 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
55 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
56 import org.apache.hadoop.yarn.util.ConverterUtils;
57 import org.junit.After;
58 import org.junit.Assert;
59 import org.junit.Before;
60 import org.junit.Test;
61 
62 public class TestNMWebServer {
63 
64   private static final File testRootDir = new File("target",
65       TestNMWebServer.class.getSimpleName());
66   private static File testLogDir = new File("target",
67       TestNMWebServer.class.getSimpleName() + "LogDir");
68 
69   @Before
setup()70   public void setup() {
71     testRootDir.mkdirs();
72     testLogDir.mkdir();
73   }
74 
75   @After
tearDown()76   public void tearDown() {
77     FileUtil.fullyDelete(testRootDir);
78     FileUtil.fullyDelete(testLogDir);
79   }
80 
startNMWebAppServer(String webAddr)81   private int startNMWebAppServer(String webAddr) {
82     Context nmContext = new NodeManager.NMContext(null, null, null, null,
83         null);
84     ResourceView resourceView = new ResourceView() {
85       @Override
86       public long getVmemAllocatedForContainers() {
87         return 0;
88       }
89       @Override
90       public long getPmemAllocatedForContainers() {
91         return 0;
92       }
93       @Override
94       public long getVCoresAllocatedForContainers() {
95         return 0;
96       }
97       @Override
98       public boolean isVmemCheckEnabled() {
99         return true;
100       }
101       @Override
102       public boolean isPmemCheckEnabled() {
103         return true;
104       }
105     };
106     Configuration conf = new Configuration();
107     conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
108     conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
109     NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
110     healthChecker.init(conf);
111     LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
112     conf.set(YarnConfiguration.NM_WEBAPP_ADDRESS, webAddr);
113     WebServer server = new WebServer(nmContext, resourceView,
114         new ApplicationACLsManager(conf), dirsHandler);
115     try {
116       server.init(conf);
117       server.start();
118       return server.getPort();
119     } finally {
120       server.stop();
121       healthChecker.stop();
122     }
123   }
124 
125   @Test
testNMWebAppWithOutPort()126   public void testNMWebAppWithOutPort() throws IOException {
127     int port = startNMWebAppServer("0.0.0.0");
128     validatePortVal(port);
129   }
130 
validatePortVal(int portVal)131   private void validatePortVal(int portVal) {
132     Assert.assertTrue("Port is not updated", portVal > 0);
133     Assert.assertTrue("Port is default "+ YarnConfiguration.DEFAULT_NM_PORT,
134                       portVal !=YarnConfiguration.DEFAULT_NM_PORT);
135   }
136 
137   @Test
testNMWebAppWithEphemeralPort()138   public void testNMWebAppWithEphemeralPort() throws IOException {
139     int port = startNMWebAppServer("0.0.0.0:0");
140     validatePortVal(port);
141   }
142 
143   @Test
testNMWebApp()144   public void testNMWebApp() throws IOException, YarnException {
145     Context nmContext = new NodeManager.NMContext(null, null, null, null,
146         null);
147     ResourceView resourceView = new ResourceView() {
148       @Override
149       public long getVmemAllocatedForContainers() {
150         return 0;
151       }
152       @Override
153       public long getPmemAllocatedForContainers() {
154         return 0;
155       }
156       @Override
157       public long getVCoresAllocatedForContainers() {
158         return 0;
159       }
160       @Override
161       public boolean isVmemCheckEnabled() {
162         return true;
163       }
164       @Override
165       public boolean isPmemCheckEnabled() {
166         return true;
167       }
168     };
169     Configuration conf = new Configuration();
170     conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
171     conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
172     NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
173     healthChecker.init(conf);
174     LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
175 
176     WebServer server = new WebServer(nmContext, resourceView,
177         new ApplicationACLsManager(conf), dirsHandler);
178     server.init(conf);
179     server.start();
180 
181     // Add an application and the corresponding containers
182     RecordFactory recordFactory =
183         RecordFactoryProvider.getRecordFactory(conf);
184     Dispatcher dispatcher = new AsyncDispatcher();
185     String user = "nobody";
186     long clusterTimeStamp = 1234;
187     ApplicationId appId =
188         BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 1);
189     Application app = mock(Application.class);
190     when(app.getUser()).thenReturn(user);
191     when(app.getAppId()).thenReturn(appId);
192     nmContext.getApplications().put(appId, app);
193     ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
194         appId, 1);
195     ContainerId container1 =
196         BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 0);
197     ContainerId container2 =
198         BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 1);
199     NodeManagerMetrics metrics = mock(NodeManagerMetrics.class);
200     NMStateStoreService stateStore = new NMNullStateStoreService();
201     for (ContainerId containerId : new ContainerId[] { container1,
202         container2}) {
203       // TODO: Use builder utils
204       ContainerLaunchContext launchContext =
205           recordFactory.newRecordInstance(ContainerLaunchContext.class);
206       long currentTime = System.currentTimeMillis();
207       Token containerToken =
208           BuilderUtils.newContainerToken(containerId, "127.0.0.1", 1234, user,
209             BuilderUtils.newResource(1024, 1), currentTime + 10000L, 123,
210             "password".getBytes(), currentTime);
211       Container container =
212           new ContainerImpl(conf, dispatcher, stateStore, launchContext,
213             null, metrics,
214             BuilderUtils.newContainerTokenIdentifier(containerToken)) {
215 
216             @Override
217             public ContainerState getContainerState() {
218               return ContainerState.RUNNING;
219             };
220           };
221       nmContext.getContainers().put(containerId, container);
222       //TODO: Gross hack. Fix in code.
223       ApplicationId applicationId =
224           containerId.getApplicationAttemptId().getApplicationId();
225       nmContext.getApplications().get(applicationId).getContainers()
226           .put(containerId, container);
227       writeContainerLogs(nmContext, containerId, dirsHandler);
228 
229     }
230     // TODO: Pull logs and test contents.
231 //    Thread.sleep(1000000);
232   }
233 
writeContainerLogs(Context nmContext, ContainerId containerId, LocalDirsHandlerService dirsHandler)234   private void writeContainerLogs(Context nmContext,
235       ContainerId containerId, LocalDirsHandlerService dirsHandler)
236         throws IOException, YarnException {
237     // ContainerLogDir should be created
238     File containerLogDir =
239         ContainerLogsUtils.getContainerLogDirs(containerId,
240             dirsHandler).get(0);
241     containerLogDir.mkdirs();
242     for (String fileType : new String[] { "stdout", "stderr", "syslog" }) {
243       Writer writer = new FileWriter(new File(containerLogDir, fileType));
244       writer.write(ConverterUtils.toString(containerId) + "\n Hello "
245           + fileType + "!");
246       writer.close();
247     }
248   }
249 }
250