docgen working with a temporary link hack.

pkg/docgen/lib/docgen.dart

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * The docgen tool takes in a library as input and produces documentation

[Andrei Mouravski, 2013/06/24 21:00:40]

**docgen** is a tool for creating machine readable representations of Dart code
metadata, including: classes, members, comments, and annotations.

docgen is run on a set of `.dart` files or directories containing `.dart` files:

    $ dart docgen.dart ...

This creates a file called `<whatever>` in your current working directory.

[janicejl, 2013/06/24 21:20:56]

Done.

  * for the library as well as all libraries it imports and uses. The tool can
  * be run by passing in the path to a .dart file like this:
  *
- *   ./dart docgen.dart path/to/file.dart
+ *   dart docgen.dart [OPTIONS] [FILE/DIR]
  *
  * This outputs information about all classes, variables, functions, and
  * methods defined in the library and its imported libraries.
  */
 library docgen;
 
-// TODO(tmandel): Use 'package:' references for imports with relative paths.
 import 'dart:io';
 import 'dart:json';
 import 'dart:async';

[Andrei Mouravski, 2013/06/24 21:00:40]

Add newline between each type of import.

[janicejl, 2013/06/24 21:20:56]

Done.

-import '../lib/dart2yaml.dart';
-import '../lib/src/dart2js_mirrors.dart';
+import 'package:args/args.dart';
+import 'package:logging/logging.dart';
 import 'package:markdown/markdown.dart' as markdown;
-import '../../args/lib/args.dart';
+import 'dart2yaml.dart';
+import '../../../sdk/lib/_internal/compiler/compiler.dart' as api;
+import '../../../sdk/lib/_internal/compiler/implementation/filenames.dart';
+import '../../../sdk/lib/_internal/compiler/implementation/mirrors/dart2js_mirror.dart'
+    as dart2js;
 import '../../../sdk/lib/_internal/compiler/implementation/mirrors/mirrors.dart';
 import '../../../sdk/lib/_internal/compiler/implementation/mirrors/mirrors_util.dart';
+import '../../../sdk/lib/_internal/compiler/implementation/source_file_provider.dart';
+

[Emily Fortuna, 2013/06/24 20:30:37]

remove extra line

[janicejl, 2013/06/24 20:56:41]

Done.

+
+/// Logger for Dart Doc Generator. 
+var logger = new Logger("Docgen");
+
+/// Unique ID, will get incremented everytime an ID is requested. 
+int _uid = 0;
+
+int getID() => _uid++;
+
+const String usage = "Usage: dart docgen.dart [OPTIONS] [fooDir/barFile]";
 
 /**
- * Entry function to create YAML documentation from Dart files.
+ * Returns a ArgParser with all the flags and options created. 
  */
-void main() {
-  // TODO(tmandel): Use args library once flags are clear.
-  Options opts = new Options();
-  Docgen docgen = new Docgen();
+ArgParser initArgParser() {
+  var parser = new ArgParser();
+  parser.addFlag("help", abbr: "h", 
+      help: "Prints help and usage information", 
+      negatable: false, 
+      callback: (help) {
+        if (help) print(parser.getUsage());
+      });
+  parser.addFlag("verbose", abbr: "v", 
+      help: "Runs docgen with logging. ", 

[Emily Fortuna, 2013/06/24 20:30:37]

Consistency! This help message ends in a period (which is fine), but none of the
others do. Also, cut the extra whitespace after the period.

[janicejl, 2013/06/24 20:56:41]

Done.

+      defaultsTo: false, negatable: false, 
+      callback: (verbose) {
+        if (verbose) logger.onRecord.listen((record) => print(record.message));
+      });
+  parser.addFlag("yaml", abbr: "y", 
+      help: "Outputs to YAML", 
+      defaultsTo: true, negatable: true);

[Emily Fortuna, 2013/06/24 20:30:37]

if the default value of negatable is true, you don't need to explicitly pass it
in each time.

[janicejl, 2013/06/24 20:56:41]

Done.

+  parser.addFlag("json", abbr: "j", 
+      help: "Outputs to JSON", 
+      defaultsTo: false, negatable: true);
+  parser.addFlag("hide-private", 
+      help: "Hides private declarations" , 
+      defaultsTo: false, negatable: false);
+  parser.addFlag("sdk", 
+      help: "Flag to parse SDK Library files", 
+      defaultsTo: true, negatable: true);
   
-  if (opts.arguments.length > 0) {
-    List<Path> libraries = [new Path(opts.arguments[0])];
-    Path sdkDirectory = new Path("../../../sdk/");
-    var workingMirrors = analyze(libraries, sdkDirectory, 
-        options: ['--preserve-comments', '--categories=Client,Server']);
-    workingMirrors.then( (MirrorSystem mirrorSystem) {
-      var mirrors = mirrorSystem.libraries.values;
-      if (mirrors.isEmpty) {
-        print("no LibraryMirrors");
-      } else {
-        docgen.libraries = mirrors;
-        docgen.documentLibraries();
-      }
-    });
+  return parser;
+}
+
+List<Path> listLibraries(List<String> args) {

[Emily Fortuna, 2013/06/24 20:30:37]

documentation for these methods?

Also, I'd just make these methods private methods to Docgen. You already pass in
the ArgResults object to Docgen when you construct it, so just use that to get
the libraries you need to document.

[janicejl, 2013/06/24 20:56:41]

Do you mean to make listLibraries a method inside Docgen? I did this when
changing to use Pathos. Should I also make the same change here?

[Emily Fortuna, 2013/06/24 21:11:36]

Okay. If you've made the change elsewhere I'll hold out for that.

[janicejl, 2013/06/24 21:20:56]

Sounds good.

+  if (args.length != 1) {
+    throw new UnsupportedError(usage);
   }
+  var libraries = new List<Path>();
+  var type = FileSystemEntity.typeSync(args[0]);
+  
+  if (type == FileSystemEntityType.NOT_FOUND) { 
+    throw new UnsupportedError("File does not exist. $usage");
+  } else if (type == FileSystemEntityType.LINK) {
+    libraries.addAll(listLibrariesFromDir(new Link(args[0]).targetSync()));
+  } else if (type == FileSystemEntityType.FILE) {
+    libraries.add(new Path(args[0]));
+    logger.info("Added to libraries: ${libraries.last.toString()}");
+  } else if (type == FileSystemEntityType.DIRECTORY) {
+    libraries.addAll(listLibrariesFromDir(args[0]));
+  }
+  return libraries;
+}
+
+List<Path> listLibrariesFromDir(String path) {
+  var libraries = new List<Path>();
+  new Directory(path).listSync(recursive: true, 
+      followLinks: true).forEach((file) {
+        if (new Path(file.path).extension == "dart") {
+          if (!file.path.contains("/packages/")) {
+            libraries.add(new Path(file.path));
+            logger.info("Added to libraries: ${libraries.last.toString()}");
+          }
+        }
+      });
+  return libraries;
 }
 
 /**
  * This class documents a list of libraries.
  */
 class Docgen {
 
   /// Libraries to be documented.
   List<LibraryMirror> _libraries;
   
   /// Saves list of libraries for Docgen object.
-  void set libraries(value) => _libraries = value;
+  void set libraries(value) {
+    _libraries = value;

[Emily Fortuna, 2013/06/24 20:30:37]

can you stick all the fields together and then declare gettors/settors in a
separate section?

[janicejl, 2013/06/24 20:56:41]

Done.

+  }
   
   /// Current library being documented to be used for comment links.
   LibraryMirror _currentLibrary;
   
   /// Current class being documented to be used for comment links.
   ClassMirror _currentClass;
   
   /// Current member being documented to be used for comment links.
   MemberMirror _currentMember;
   
-  /// Should the output file type be JSON?
-  // TODO(tmandel): Add flag to allow for output to JSON.
-  bool outputToJson = false;
-  
   /// Resolves reference links
   markdown.Resolver linkResolver;
   
+  /// Should the output file type be YAML?

[Emily Fortuna, 2013/06/24 20:30:37]

If the value held is obvious by the variable name, you probably don't need to
document. Also, don't have documentation comments that are questions.

[janicejl, 2013/06/24 20:56:41]

Done.

+  bool outputToYaml;
+  /// Should the output file type be JSON?
+  bool outputToJson;
+  /// Should the output file hide private declarations?
+  bool hidePrivate;
+  /// Should the output include SDK libraries?
+  bool sdk;
+  
   /**
    * Docgen constructor initializes the link resolver for markdown parsing.
+   * Also initializes the command line arguments. 
    */
-  Docgen() {
+  Docgen(ArgResults argResults) {  
+    outputToYaml = argResults["yaml"];
+    outputToJson = argResults["json"];
+    hidePrivate = argResults["hide-private"];
+    sdk = argResults["sdk"];
+    
     this.linkResolver = (name) => 
         fixReference(name, _currentLibrary, _currentClass, _currentMember);
   }
   
   /**
+   * Analyzes set of libraries by getting a mirror system and triggers the 
+   * documentation of the libraries. 
+   */
+  void analyze(List<Path> libraries) {
+    // DART_SDK should be set to the root of the SDK library. 
+    var sdkRoot = Platform.environment["DART_SDK"];
+    if (sdkRoot != null) {
+      logger.info("Using DART_SDK to find SDK at $sdkRoot");
+      sdkRoot = new Path(sdkRoot);
+    } else {
+      // If DART_SDK is not defined in the environment, 
+      // assuming the dart executable is from the Dart SDK folder inside bin. 
+      sdkRoot = new Path(new Options().executable).directoryPath
+          .directoryPath;
+      logger.info("SDK Root: ${sdkRoot.toString()}");
+    }
+    
+    Path packageDir = libraries.last.directoryPath.append("packages");
+    logger.info("Package Root: ${packageDir.toString()}");
+    getMirrorSystem(libraries, sdkRoot, 
+        packageRoot: packageDir).then((MirrorSystem mirrorSystem) {
+          if (mirrorSystem.libraries.values.isEmpty) {
+            throw new UnsupportedError("No Library Mirrors.");
+          } 
+          this.libraries = mirrorSystem.libraries.values;
+          documentLibraries();
+        });
+  }
+  
+  /**
+   * Analyzes set of libraries and provides a mirror system which can be used 
+   * for static inspection of the source code.
+   */
+  Future<MirrorSystem> getMirrorSystem(List<Path> libraries,
+        Path libraryRoot, {Path packageRoot}) {
+    SourceFileProvider provider = new SourceFileProvider();
+    api.DiagnosticHandler diagnosticHandler =
+          new FormattingDiagnosticHandler(provider).diagnosticHandler;
+    Uri libraryUri = currentDirectory.resolve(appendSlash('$libraryRoot'));
+    Uri packageUri = null;
+    if (packageRoot != null) {
+      packageUri = currentDirectory.resolve(appendSlash('$packageRoot'));
+    }
+    List<Uri> librariesUri = <Uri>[];
+    libraries.forEach((library) {
+      librariesUri.add(currentDirectory.resolve(library.toString()));
+    });
+    return dart2js.analyze(librariesUri, libraryUri, packageUri,
+        provider.readStringFromUri, diagnosticHandler,
+        ['--preserve-comments', '--categories=Client,Server']);
+  }
+  
+  /**
    * Creates documentation for filtered libraries.
    */
   void documentLibraries() {
-    //TODO(tmandel): Filter libraries and determine output type using flags.
     _libraries.forEach((library) {
-      _currentLibrary = library;
-      var result = new Library(library.qualifiedName, _getComment(library),
-          _getVariables(library.variables), _getMethods(library.functions),
-          _getClasses(library.classes));
-      if (outputToJson) {
-        _writeToFile(stringify(result.toMap()), "${result.name}.json");
-      } else {
-        _writeToFile(getYamlString(result.toMap()), "${result.name}.yaml");
-      }
+      // Files belonging to the SDK have a uri that begins with "dart:".
+      if (sdk || !library.uri.toString().startsWith("dart:")) {
+        _currentLibrary = library;
+        var result = new Library(library.qualifiedName, _getComment(library),
+            _getVariables(library.variables), _getMethods(library.functions),
+            _getClasses(library.classes), getID());
+        if (outputToJson) {
+          _writeToFile(stringify(result.toMap()), "${result.name}.json");
+        } 
+        if (outputToYaml) {
+          _writeToFile(getYamlString(result.toMap()), "${result.name}.yaml");
+        }
+     }
     });
    }
   
   /**
    * Returns any documentation comments associated with a mirror with
    * simple markdown converted to html.
    */
   String _getComment(DeclarationMirror mirror) {
     String commentText;
     mirror.metadata.forEach((metadata) {
       if (metadata is CommentInstanceMirror) {
         CommentInstanceMirror comment = metadata;
         if (comment.isDocComment) {
           if (commentText == null) {
             commentText = comment.trimmedText;
           } else {
             commentText = "$commentText ${comment.trimmedText}";
           }
         } 
       }
     });
-    return commentText == null ? "" : 
-      markdown.markdownToHtml(commentText.trim(), linkResolver: linkResolver);
+    commentText = commentText == null ? "" : 
+        markdown.markdownToHtml(commentText.trim(), linkResolver: linkResolver)
+        .replaceAll("\n", "");
+    return commentText;
   }
 
   /**
    * Converts all [_] references in comments to <code>_</code>.
    */
   // TODO(tmandel): Create proper links for [_] style markdown based
   // on scope once layout of viewer is finished.
   markdown.Node fixReference(String name, LibraryMirror currentLibrary, 
       ClassMirror currentClass, MemberMirror currentMember) {
     return new markdown.Element.text('code', name);
   }
   
   /**
    * Returns a map of [Variable] objects constructed from inputted mirrors.
    */
   Map<String, Variable> _getVariables(Map<String, VariableMirror> mirrorMap) {
     var data = {};
     mirrorMap.forEach((String mirrorName, VariableMirror mirror) {
-      _currentMember = mirror;
-      data[mirrorName] = new Variable(mirrorName, mirror.isFinal,
-          mirror.isStatic, mirror.type.toString(), _getComment(mirror));
+      if (!hidePrivate || !mirror.isPrivate) {
+        _currentMember = mirror;
+        data[mirrorName] = new Variable(mirrorName, mirror.isFinal,
+            mirror.isStatic, mirror.type.toString(), _getComment(mirror), 
+            getID());
+      }
     });
     return data;
   }
   
   /**
    * Returns a map of [Method] objects constructed from inputted mirrors.
    */
   Map<String, Method> _getMethods(Map<String, MethodMirror> mirrorMap) {
     var data = {};
     mirrorMap.forEach((String mirrorName, MethodMirror mirror) {
-      _currentMember = mirror;
-      data[mirrorName] = new Method(mirrorName, mirror.isSetter,
-          mirror.isGetter, mirror.isConstructor, mirror.isOperator, 
-          mirror.isStatic, mirror.returnType.toString(), _getComment(mirror),
-          _getParameters(mirror.parameters));
+      if (!hidePrivate || !mirror.isPrivate) {
+        _currentMember = mirror;
+        data[mirrorName] = new Method(mirrorName, mirror.isSetter,
+            mirror.isGetter, mirror.isConstructor, mirror.isOperator, 
+            mirror.isStatic, mirror.returnType.toString(), _getComment(mirror),
+            _getParameters(mirror.parameters), getID());
+      }
     });
     return data;
   } 
   
   /**
    * Returns a map of [Class] objects constructed from inputted mirrors.
    */
   Map<String, Class> _getClasses(Map<String, ClassMirror> mirrorMap) {
     var data = {};
     mirrorMap.forEach((String mirrorName, ClassMirror mirror) {
-      _currentClass = mirror;
-      var superclass;
-      if (mirror.superclass != null) {
-        superclass = mirror.superclass.qualifiedName;
+      if (!hidePrivate || !mirror.isPrivate) {
+        _currentClass = mirror;
+        var superclass = (mirror.superclass != null) ? 
+            mirror.superclass.qualifiedName : "";
+        var interfaces = 
+            mirror.superinterfaces.map((interface) => interface.qualifiedName);
+        data[mirrorName] = new Class(mirrorName, superclass, mirror.isAbstract,
+            mirror.isTypedef, _getComment(mirror), interfaces.toList(),
+            _getVariables(mirror.variables), _getMethods(mirror.methods), 
+            getID());
       }
-      var interfaces = 
-          mirror.superinterfaces.map((interface) => interface.qualifiedName);
-      data[mirrorName] = new Class(mirrorName, superclass, mirror.isAbstract,
-          mirror.isTypedef, _getComment(mirror), interfaces,
-          _getVariables(mirror.variables), _getMethods(mirror.methods));
     });
     return data;
   }
   
   /**
    * Returns a map of [Parameter] objects constructed from inputted mirrors.
    */
   Map<String, Parameter> _getParameters(List<ParameterMirror> mirrorList) {
     var data = {};
     mirrorList.forEach((ParameterMirror mirror) {
       _currentMember = mirror;
       data[mirror.simpleName] = new Parameter(mirror.simpleName, 
           mirror.isOptional, mirror.isNamed, mirror.hasDefaultValue, 
-          mirror.type.toString(), mirror.defaultValue);
+          mirror.type.toString(), mirror.defaultValue, getID());
     });
     return data;
   }
 }
 
 /**
  * Transforms the map by calling toMap on each value in it.
  */
 Map recurseMap(Map inputMap) {
   var outputMap = {};
   inputMap.forEach((key, value) {
     outputMap[key] = value.toMap();
   });
   return outputMap;
 }
 
 /**
  * A class containing contents of a Dart library.
  */
 class Library {
   
+  /// Unique ID number for resolving links. 
+  int id;
+  
   /// Documentation comment with converted markdown.
   String comment;
   
   /// Top-level variables in the library.
   Map<String, Variable> variables;
   
   /// Top-level functions in the library.
   Map<String, Method> functions;
   
   /// Classes defined within the library
   Map<String, Class> classes;
   
   String name;
   
   Library(this.name, this.comment, this.variables, 
-      this.functions, this.classes);
+      this.functions, this.classes, this.id);
   
   /// Generates a map describing the [Library] object.
   Map toMap() {
     var libraryMap = {};
+    libraryMap["id"] = id;
     libraryMap["name"] = name;
     libraryMap["comment"] = comment;
     libraryMap["variables"] = recurseMap(variables);
     libraryMap["functions"] = recurseMap(functions);
     libraryMap["classes"] = recurseMap(classes);
     return libraryMap;
   }
 }
 
 /**
  * A class containing contents of a Dart class.
  */
 // TODO(tmandel): Figure out how to do typedefs (what is needed)
 class Class {
   
+  /// Unique ID number for resolving links. 
+  int id;
+  
   /// Documentation comment with converted markdown.
   String comment;
   
   /// List of the names of interfaces that this class implements.
   List<String> interfaces;
   
   /// Top-level variables in the class.
   Map<String, Variable> variables;
   
   /// Methods in the class.
   Map<String, Method> methods;
   
   String name;
   String superclass;
   bool isAbstract;
   bool isTypedef;
  
   Class(this.name, this.superclass, this.isAbstract, this.isTypedef,
-      this.comment, this.interfaces, this.variables, this.methods); 
+      this.comment, this.interfaces, this.variables, this.methods, this.id);
   
   /// Generates a map describing the [Class] object.
   Map toMap() {
     var classMap = {};
+    classMap["id"] = id;
     classMap["name"] = name;
     classMap["comment"] = comment;
     classMap["superclass"] = superclass;
     classMap["abstract"] = isAbstract.toString();
     classMap["typedef"] = isTypedef.toString();
     classMap["implements"] = new List.from(interfaces);
     classMap["variables"] = recurseMap(variables);
     classMap["methods"] = recurseMap(methods);
     return classMap;
   }
 }
 
 /**
  * A class containing properties of a Dart variable.
  */
 class Variable {
   
+  /// Unique ID number for resolving links. 
+  int id;
+  
   /// Documentation comment with converted markdown.
   String comment;
   
   String name;
   bool isFinal;
   bool isStatic;
   String type;
   
-  Variable(this.name, this.isFinal, this.isStatic, this.type, this.comment);
+  Variable(this.name, this.isFinal, this.isStatic, this.type, 
+      this.comment, this.id);
   
   /// Generates a map describing the [Variable] object.
   Map toMap() {
     var variableMap = {};
+    variableMap["id"] = id;
     variableMap["name"] = name;
     variableMap["comment"] = comment;
     variableMap["final"] = isFinal.toString();
     variableMap["static"] = isStatic.toString();
     variableMap["type"] = type;
     return variableMap;
   }
 }
 
 /**
  * A class containing properties of a Dart method.
  */
 class Method {
   
+  /// Unique ID number for resolving links. 
+  int id;
+  
   /// Documentation comment with converted markdown.
   String comment;
   
   /// Parameters for this method.
   Map<String, Parameter> parameters;
   
   String name;
   bool isSetter;
   bool isGetter;
   bool isConstructor;
   bool isOperator;
   bool isStatic;
   String returnType;
   
   Method(this.name, this.isSetter, this.isGetter, this.isConstructor,
       this.isOperator, this.isStatic, this.returnType, this.comment,
-      this.parameters);
+      this.parameters, this.id);
   
   /// Generates a map describing the [Method] object.
   Map toMap() {
     var methodMap = {};
+    methodMap["id"] = id;
     methodMap["name"] = name;
     methodMap["comment"] = comment;
     methodMap["type"] = isSetter ? "setter" : isGetter ? "getter" :
       isOperator ? "operator" : isConstructor ? "constructor" : "method";
     methodMap["static"] = isStatic.toString();
     methodMap["return"] = returnType;
     methodMap["parameters"] = recurseMap(parameters);
     return methodMap;
   }  
 }
 
 /**
  * A class containing properties of a Dart method/function parameter.
  */
 class Parameter {
   
+  /// Unique ID number for resolving links. 
+  int id;
+  
   String name;
   bool isOptional;
   bool isNamed;
   bool hasDefaultValue;
   String type;
   String defaultValue;
   
   Parameter(this.name, this.isOptional, this.isNamed, this.hasDefaultValue,
-      this.type, this.defaultValue);
+      this.type, this.defaultValue, this.id);
   
   /// Generates a map describing the [Parameter] object.
   Map toMap() {
     var parameterMap = {};
+    parameterMap["id"] = id;
     parameterMap["name"] = name;
     parameterMap["optional"] = isOptional.toString();
     parameterMap["named"] = isNamed.toString();
     parameterMap["default"] = hasDefaultValue.toString();
     parameterMap["type"] = type;
     parameterMap["value"] = defaultValue;
     return parameterMap;
   } 
 }
 
 /**
  * Writes text to a file in the 'docs' directory.
  */
 void _writeToFile(String text, String filename) {
   Directory dir = new Directory('docs');
   if (!dir.existsSync()) {
     dir.createSync();
   }
   File file = new File('docs/$filename');
-  if (!file.exists()) {
+  if (!file.existsSync()) {
     file.createSync();
   }
   file.openSync();
   file.writeAsString(text);
 }