Commit fdc1fd34 authored by mohammad.salama's avatar mohammad.salama

Initial commit - no logging or ssh

parents
# Default ignored files
/shelf/
/workspace.xml
<component name="ArtifactManager">
<artifact type="jar" build-on-make="true" name="TransientWorker:jar">
<output-path>$PROJECT_DIR$/out/artifacts/TransientWorker_jar</output-path>
<root id="archive" name="Registration&amp;Discovery-AutoHealer.jar">
<element id="module-output" name="Registration&amp;Discovery-AutoHealer" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/io/netty/netty-transport/4.1.94.Final/netty-transport-4.1.94.Final.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/io/netty/netty-transport-classes-epoll/4.1.94.Final/netty-transport-classes-epoll-4.1.94.Final.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/io/netty/netty-tcnative-boringssl-static/2.0.61.Final/netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/io/netty/netty-transport-native-epoll/4.1.94.Final/netty-transport-native-epoll-4.1.94.Final-linux-x86_64.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/slf4j/slf4j-api/1.7.30/slf4j-api-1.7.30.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/io/netty/netty-codec/4.1.94.Final/netty-codec-4.1.94.Final.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/io/netty/netty-resolver/4.1.94.Final/netty-resolver-4.1.94.Final.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/io/netty/netty-handler/4.1.94.Final/netty-handler-4.1.94.Final.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apache/zookeeper/zookeeper-jute/3.9.1/zookeeper-jute-3.9.1.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/io/netty/netty-tcnative-classes/2.0.61.Final/netty-tcnative-classes-2.0.61.Final.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/ch/qos/logback/logback-classic/1.2.10/logback-classic-1.2.10.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/commons-io/commons-io/2.11.0/commons-io-2.11.0.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/io/netty/netty-transport-native-unix-common/4.1.94.Final/netty-transport-native-unix-common-4.1.94.Final.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/io/netty/netty-common/4.1.94.Final/netty-common-4.1.94.Final.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/io/netty/netty-tcnative-boringssl-static/2.0.61.Final/netty-tcnative-boringssl-static-2.0.61.Final.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/io/netty/netty-tcnative-boringssl-static/2.0.61.Final/netty-tcnative-boringssl-static-2.0.61.Final-linux-aarch_64.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/ch/qos/logback/logback-core/1.2.10/logback-core-1.2.10.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apache/zookeeper/zookeeper/3.9.1/zookeeper-3.9.1.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/io/netty/netty-tcnative-boringssl-static/2.0.61.Final/netty-tcnative-boringssl-static-2.0.61.Final-osx-x86_64.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apache/yetus/audience-annotations/0.12.0/audience-annotations-0.12.0.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/io/netty/netty-tcnative-boringssl-static/2.0.61.Final/netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/io/netty/netty-buffer/4.1.94.Final/netty-buffer-4.1.94.Final.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/io/netty/netty-tcnative-boringssl-static/2.0.61.Final/netty-tcnative-boringssl-static-2.0.61.Final-osx-aarch_64.jar" path-in-jar="/" />
</root>
</artifact>
</component>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CompilerConfiguration">
<annotationProcessing>
<profile name="Maven default annotation processors profile" enabled="true">
<sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" />
<module name="Registration&amp;Discovery-AutoHealer" />
<module name="Service-Registration-and-Discovery" />
</profile>
</annotationProcessing>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding">
<file url="file://$PROJECT_DIR$/src/main/java" charset="UTF-8" />
<file url="file://$PROJECT_DIR$/src/main/resources" charset="UTF-8" />
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="RemoteRepositoriesConfiguration">
<remote-repository>
<option name="id" value="central" />
<option name="name" value="Central Repository" />
<option name="url" value="https://repo.maven.apache.org/maven2" />
</remote-repository>
<remote-repository>
<option name="id" value="central" />
<option name="name" value="Maven Central repository" />
<option name="url" value="https://repo1.maven.org/maven2" />
</remote-repository>
<remote-repository>
<option name="id" value="jboss.community" />
<option name="name" value="JBoss Community repository" />
<option name="url" value="https://repository.jboss.org/nexus/content/repositories/public/" />
</remote-repository>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ExternalStorageConfigurationManager" enabled="true" />
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_17" default="true" project-jdk-name="17 (2)" project-jdk-type="JavaSDK" />
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="" vcs="Git" />
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.example</groupId>
<artifactId>Service-Registration-and-Discovery</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.11.0</version>
<configuration>
<source>17</source>
<target>17</target>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.9.1</version>
</dependency>
</dependencies>
</project>
\ No newline at end of file
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
public class Application implements Watcher {
private static final String address = "192.168.184.10:2181";
private static final int SESSION_TIMEOUT = 3000; //dead client
private static final int DEFAULT_PORT = 8080;
private ZooKeeper zooKeeper;
public static int numberOfInstances;
public static String pathToFile = "";
public static void main(String[] args) throws IOException, InterruptedException, KeeperException {
int currentServerPort = args.length == 3 ? Integer.parseInt(args[0]) : DEFAULT_PORT;
numberOfInstances = args.length == 3 ? Integer.parseInt(args[1]) : 100;
pathToFile = args.length == 3 ? args[2] : "D:\\HIAST\\FIY\\FS\\Distributed Systems\\Lab\\6\\DS-06\\Registration&Discovery-AutoHealer\\out\\artifacts\\TransientWorker_jar\\Registration&Discovery-AutoHealer.jar";
Application application = new Application();
ZooKeeper zooKeeper = application.connectToZookeeper();
ServiceRegistry serviceRegistry = new ServiceRegistry(zooKeeper , numberOfInstances , pathToFile);
OnElectionAction onElectionAction = new OnElectionAction(serviceRegistry, currentServerPort);
LeaderElection leaderElection = new LeaderElection(zooKeeper, onElectionAction);
leaderElection.volunteerForLeadership();
leaderElection.reelectLeader();
application.run();
application.close();
}
public ZooKeeper connectToZookeeper() throws IOException {
this.zooKeeper = new ZooKeeper(address, SESSION_TIMEOUT, this);
return zooKeeper;
}
public void run() throws InterruptedException {
synchronized (zooKeeper) {
zooKeeper.wait();
}
}
private void close() throws InterruptedException {
this.zooKeeper.close();
}
@Override
public void process(WatchedEvent watchedEvent) {
switch (watchedEvent.getType()) {
case None:
if (watchedEvent.getState() == Event.KeeperState.SyncConnected) {
System.out.println("Successfully connected to Zookeeper");
} else if (watchedEvent.getState() == Event.KeeperState.Disconnected) {
synchronized (zooKeeper) {
System.out.println("Disconnected from Zookeeper");
zooKeeper.notifyAll();
}
} else if (watchedEvent.getState() == Event.KeeperState.Closed) {
System.out.println("Closed Successfully");
}
break;
}
}
}
import org.apache.zookeeper.*;
import org.apache.zookeeper.data.Stat;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Collections;
import java.util.List;
public class LeaderElection implements Watcher {
private static final String AUTOHEALER_ZNODES_PATH = "/physical_nodes";
private String currentZnodeName;
private ZooKeeper zooKeeper;
private OnElectionCallback onElectionCallback;
public LeaderElection(ZooKeeper zooKeeper, OnElectionCallback onElectionCallback) {
this.zooKeeper = zooKeeper;
this.onElectionCallback = onElectionCallback;
}
public void volunteerForLeadership() throws InterruptedException, KeeperException {
String znodePrefix = AUTOHEALER_ZNODES_PATH + "/node_";
String znodeFullPath = zooKeeper.create(znodePrefix, new byte[]{}, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL);
System.out.println(znodeFullPath);
this.currentZnodeName = znodeFullPath.replace(AUTOHEALER_ZNODES_PATH + "/", "");
}
public void reelectLeader() throws InterruptedException, KeeperException {
String predecessorName = "";
Stat predecessorStat = null;
//this while to guarantee get predecessor even if it deleted just before zookeeper.exist
while (predecessorStat == null)
{
List<String> children = zooKeeper.getChildren(AUTOHEALER_ZNODES_PATH, false);
Collections.sort(children);
String smallestChild = children.get(0); //the first element
if (smallestChild.equals(currentZnodeName)) {
System.out.println("I'm a leader");
try
{
String IP = InetAddress.getLocalHost().getHostAddress();
String username = System.getProperty("user.name")+"@"+IP;
zooKeeper.setData(AUTOHEALER_ZNODES_PATH + "/"+currentZnodeName , username.getBytes() , -1);
} catch (UnknownHostException e) {
throw new RuntimeException(e);
}
onElectionCallback.onElectedToBeLeader();
return;
}
else {
System.out.println("I'm not a leader");
int predecessorIndex = children.indexOf(currentZnodeName) - 1;
predecessorName = children.get(predecessorIndex);
predecessorStat = zooKeeper.exists(AUTOHEALER_ZNODES_PATH + "/" + predecessorName, this);
}
}
onElectionCallback.onWorker();
System.out.println("Watching znode " + predecessorName);
System.out.println();
}
@Override
public void process(WatchedEvent watchedEvent) {
switch (watchedEvent.getType())
{
case NodeDeleted:
try
{
reelectLeader();
}
catch (InterruptedException | KeeperException e) {
throw new RuntimeException(e);
}
break;
}
}
}
import java.util.*;
public class NodeSorting
{
private static class Pair
{
String node;
int tasksNo;
// Constructor
public Pair(String x, int y) {
this.node = x;
this.tasksNo = y;
}
}
public static List<String> sort(List<byte[]> workers , List<String> Nodes)
{
Map<String , Integer> map = new HashMap<>();
for (byte[] worker : workers)
{
String s = new String(worker);
int x = map.getOrDefault(s , 0);
map.put(s , x+1);
}
for (String node : Nodes)
{
int x = map.getOrDefault(node , 0);
map.put(node , x+1);
}
List<Pair> temp = new ArrayList<>();
for (Map.Entry<String , Integer> el : map.entrySet())
{
Pair pair = new Pair(el.getKey() , el.getValue());
temp.add(pair);
}
Comparator<Pair> comparator = new Comparator<>()
{
@Override
public int compare(Pair p1, Pair p2)
{
return p1.tasksNo - p2.tasksNo;
}
};
List<String> ans = new ArrayList<>();
Collections.sort(temp, comparator);
for (Pair pair : temp)
{
ans.add(pair.node);
}
return ans;
}
}
import org.apache.zookeeper.KeeperException;
import java.net.InetAddress;
import java.net.UnknownHostException;
public class OnElectionAction implements OnElectionCallback
{
private final ServiceRegistry serviceRegistry;
private final int port;
public OnElectionAction(ServiceRegistry serviceRegistry, int port)
{
this.serviceRegistry = serviceRegistry;
this.port = port;
}
@Override
public void onElectedToBeLeader()
{
serviceRegistry.unregisterFromCluster();
serviceRegistry.registerForUpdates();
}
@Override
public void onWorker()
{
try
{
String IP = InetAddress.getLocalHost().getHostAddress();
String username = System.getProperty("user.name");
serviceRegistry.registerToCluster(username+"@"+IP);
}
catch (InterruptedException | KeeperException | UnknownHostException e)
{
e.printStackTrace();
}
}
}
public interface OnElectionCallback {
void onElectedToBeLeader();
void onWorker();
}
import org.apache.zookeeper.*;
import org.apache.zookeeper.data.Stat;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public class ServiceRegistry implements Watcher
{
private static final String WORKERS_ZNODES_PATH = "/workers";
private static final String AUTOHEALER_ZNODES_PATH = "/physical_nodes";
private String pathToProgram = "";
private final ZooKeeper zooKeeper;
private String currentZnode = null;
private List<String> allServiceAddresses = null;
private int numberOfInstances;
public ServiceRegistry(ZooKeeper zooKeeper , int numberOfInstances , String pathToProgram)
{
this.zooKeeper = zooKeeper;
this.numberOfInstances = numberOfInstances;
this.pathToProgram = pathToProgram;
createServiceRegistryZnode();
}
private void createServiceRegistryZnode()
{
try
{
if (zooKeeper.exists(WORKERS_ZNODES_PATH, false) == null) {
zooKeeper.create(WORKERS_ZNODES_PATH, new byte[]{}, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
}
if (zooKeeper.exists(AUTOHEALER_ZNODES_PATH, false) == null) {
zooKeeper.create(AUTOHEALER_ZNODES_PATH, new byte[]{}, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
}
}
catch (KeeperException | InterruptedException e) {
e.printStackTrace();
}
}
public void registerToCluster(String metadata) throws KeeperException, InterruptedException {
System.out.println(metadata);
if (this.currentZnode != null)
{
zooKeeper.setData(AUTOHEALER_ZNODES_PATH + "/"+currentZnode , metadata.getBytes() , -1);
System.out.println("Already registered to service registry");
return;
}
this.currentZnode = zooKeeper.create(AUTOHEALER_ZNODES_PATH + "/node_", metadata.getBytes(),
ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL);
System.out.println("Registered to service registry");
}
public void registerForUpdates() {
try
{
masterJob();
}
catch (KeeperException | InterruptedException | IOException e) {
e.printStackTrace();
}
}
public void unregisterFromCluster()
{
try
{
if (currentZnode != null && zooKeeper.exists(currentZnode, false) != null) {
zooKeeper.delete(currentZnode, -1);
}
}
catch (KeeperException | InterruptedException e)
{
e.printStackTrace();
}
}
private void masterJob() throws InterruptedException, KeeperException, IOException {
updateAddresses();
launchWorkersIfNecessary();
}
private synchronized void updateAddresses() throws KeeperException, InterruptedException
{
List<String> workerZnodes = zooKeeper.getChildren(AUTOHEALER_ZNODES_PATH, this);
List<String> addresses = new ArrayList<>(workerZnodes.size());
for (String workerZnode : workerZnodes) {
String workerFullPath = AUTOHEALER_ZNODES_PATH + "/" + workerZnode;
Stat stat = zooKeeper.exists(workerFullPath, false);
if (stat == null) {
continue;
}
byte[] addressBytes = zooKeeper.getData(workerFullPath, false, stat);
String address = new String(addressBytes);
addresses.add(address);
}
this.allServiceAddresses = Collections.unmodifiableList(addresses);
System.out.println("The cluster addresses are: " + this.allServiceAddresses);
}
/**
* this method misses the implementation of ssh to use startNewWorker
* */
private void launchWorkersIfNecessary() throws KeeperException, InterruptedException, IOException {
List<String> physicalZnodes = zooKeeper.getChildren(AUTOHEALER_ZNODES_PATH, false);
List<String> workers = zooKeeper.getChildren(WORKERS_ZNODES_PATH , this);
int neededInstances = numberOfInstances - workers.size();
int index= 0;
int code = 1;
List<String> sortedWorkers = NodeSorting.sort(getOriginalNodes(workers) , physicalZnodes);
while (neededInstances > 0 && sortedWorkers.size()>0)
{
Stat stat = zooKeeper.exists(AUTOHEALER_ZNODES_PATH+"/"+sortedWorkers.get(index) ,false);
if (stat == null)
{
code++;
if (code > sortedWorkers.size())
{
break;
}
continue;
}
/*
* ssh to execute pathToProgram in the coherent IP for the node
* */
startNewWorker(sortedWorkers.get(index));
neededInstances--;
index = (index+1)%sortedWorkers.size();
/// System.out.println("Worker " + uuuu);
}
}
private List<byte[]> getOriginalNodes(List<String> workers) throws InterruptedException, KeeperException {
List<byte[]> ans = new ArrayList<>();
for (String worker : workers)
{
Stat stat = zooKeeper.exists(WORKERS_ZNODES_PATH+"/"+worker ,false);
if (stat == null) continue;
ans.add(zooKeeper.getData(WORKERS_ZNODES_PATH+"/"+worker,false,stat ));
}
return ans;
}
private void startNewWorker(String physicalNode) throws IOException, InterruptedException, KeeperException
{
File file = new File(pathToProgram);
String command = "java -Dorg.slf4j.simpleLogger.defaultLogLevel=off -jar " + file.getName() + " " + physicalNode;
String ip = new String(zooKeeper.getData(AUTOHEALER_ZNODES_PATH+"/"+physicalNode , false, null));
System.out.println("Sendig job to " + ip);
/*
* must execute command over ssh here !!
* */
Runtime.getRuntime().exec(command, null, file.getParentFile());
}
@Override
public void process(WatchedEvent watchedEvent)
{
try {
masterJob();
} catch (InterruptedException | KeeperException | IOException e) {
throw new RuntimeException(e);
}
}
}
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.ZooKeeper;
import java.io.IOException;
import java.util.Random;
import java.util.concurrent.locks.LockSupport;
public class TransientWorker
{
private static final String ZOOKEEPER_ADDRESS = "192.168.184.10:2181";
private static final int SESSION_TIMEOUT = 3000;
// Parent Znode where each worker stores an ephemeral child to indicate it is alive
private static final String WORKERS_ZNODES_PATH = "/workers";
private static final float CHANCE_TO_FAIL = 0.001F;
private final Random random = new Random();
private ZooKeeper zooKeeper;
private String myName = "";
public void connectToZookeeper() throws IOException {
this.zooKeeper = new ZooKeeper(ZOOKEEPER_ADDRESS, SESSION_TIMEOUT, event -> {
});
}
public void work(String nodeNum) throws KeeperException, InterruptedException {
addChildZnode(nodeNum);
while (true)
{
System.out.println(myName + " is Working...");
LockSupport.parkNanos(1000L);
if (random.nextFloat() < CHANCE_TO_FAIL) {
System.out.println(myName + " encountered Critical error happened");
throw new RuntimeException(myName + " : Oops");
}
}
}
private void addChildZnode(String nodeNumber) throws KeeperException, InterruptedException {
myName= zooKeeper.create(WORKERS_ZNODES_PATH + "/worker_",
nodeNumber.getBytes(),
ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.EPHEMERAL_SEQUENTIAL);
}
}
import java.io.IOException;
public class TransientWorkerApplication
{
public static void main(String[] args)
{
TransientWorker worker = new TransientWorker();
String nodeNum = args[0];
try
{
worker.connectToZookeeper();
} catch (IOException e) {
throw new RuntimeException(e);
}
try
{
worker.work(nodeNum);
}
catch (Exception e) {
}
}
}
Manifest-Version: 1.0
Main-Class: TransientWorkerApplication
<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="WARN">
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
</Console>
</Appenders>
<Loggers>
<Root level="WARN">
<AppenderRef ref="Console"/>
</Root>
</Loggers>
</Configuration>
\ No newline at end of file
Manifest-Version: 1.0
Main-Class: TransientWorkerApplication
<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="WARN">
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
</Console>
</Appenders>
<Loggers>
<Root level="WARN">
<AppenderRef ref="Console"/>
</Root>
</Loggers>
</Configuration>
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment